You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by kt...@apache.org on 2016/07/15 22:07:49 UTC
[01/10] incubator-fluo-recipes git commit: Updated package names in
core module
Repository: incubator-fluo-recipes
Updated Branches:
refs/heads/master a8b85f332 -> 22354d0f7
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/CollisionFreeMapIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/CollisionFreeMapIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/CollisionFreeMapIT.java
deleted file mode 100644
index 0743497..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/CollisionFreeMapIT.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.commons.io.FileUtils;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.client.LoaderExecutor;
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.ObserverConfiguration;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.api.mini.MiniFluo;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CollisionFreeMapIT {
-
- private MiniFluo miniFluo;
-
- private CollisionFreeMap<String, Long> wcMap;
-
- static final String MAP_ID = "wcm";
-
- @Before
- public void setUpFluo() throws Exception {
- FileUtils.deleteQuietly(new File("target/mini"));
-
- FluoConfiguration props = new FluoConfiguration();
- props.setApplicationName("eqt");
- props.setWorkerThreads(20);
- props.setMiniDataDir("target/mini");
-
- props.addObserver(new ObserverConfiguration(DocumentObserver.class.getName()));
-
- SimpleSerializer.setSetserlializer(props, TestSerializer.class);
-
- CollisionFreeMap.configure(props, new CollisionFreeMap.Options(MAP_ID, WordCountCombiner.class,
- WordCountObserver.class, String.class, Long.class, 17));
-
- miniFluo = FluoFactory.newMiniFluo(props);
-
- wcMap = CollisionFreeMap.getInstance(MAP_ID, props.getAppConfiguration());
- }
-
- @After
- public void tearDownFluo() throws Exception {
- if (miniFluo != null) {
- miniFluo.close();
- }
- }
-
- private Map<String, Long> getComputedWordCounts(FluoClient fc) {
- Map<String, Long> counts = new HashMap<>();
-
- try (Snapshot snap = fc.newSnapshot()) {
- RowIterator scanner = snap.get(new ScannerConfiguration().setSpan(Span.prefix("iwc:")));
- while (scanner.hasNext()) {
- Entry<Bytes, ColumnIterator> row = scanner.next();
-
- String[] tokens = row.getKey().toString().split(":");
- String word = tokens[2];
- Long count = Long.valueOf(tokens[1]);
-
- Assert.assertFalse("Word seen twice in index " + word, counts.containsKey(word));
-
- counts.put(word, count);
- }
- }
-
- return counts;
- }
-
- private Map<String, Long> computeWordCounts(FluoClient fc) {
- Map<String, Long> counts = new HashMap<>();
-
- try (Snapshot snap = fc.newSnapshot()) {
- RowIterator scanner =
- snap.get(new ScannerConfiguration().setSpan(Span.prefix("d:")).fetchColumn(
- Bytes.of("content"), Bytes.of("current")));
- while (scanner.hasNext()) {
- Entry<Bytes, ColumnIterator> row = scanner.next();
-
- ColumnIterator colIter = row.getValue();
-
- while (colIter.hasNext()) {
- Entry<Column, Bytes> entry = colIter.next();
-
- String[] words = entry.getValue().toString().split("\\s+");
- for (String word : words) {
- if (word.isEmpty()) {
- continue;
- }
-
- counts.merge(word, 1L, Long::sum);
- }
- }
- }
- }
-
- return counts;
- }
-
- @Test
- public void testGet() {
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- try (Transaction tx = fc.newTransaction()) {
- wcMap.update(tx, ImmutableMap.of("cat", 2L, "dog", 5L));
- tx.commit();
- }
-
- try (Transaction tx = fc.newTransaction()) {
- wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", 1L));
- tx.commit();
- }
-
- try (Transaction tx = fc.newTransaction()) {
- wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", 1L, "fish", 2L));
- tx.commit();
- }
-
- // try reading possibly before observer combines... will either see outstanding updates or a
- // current value
- try (Snapshot snap = fc.newSnapshot()) {
- Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
- Assert.assertEquals((Long) 7L, wcMap.get(snap, "dog"));
- Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
- }
-
- miniFluo.waitForObservers();
-
- // in this case there should be no updates, only a current value
- try (Snapshot snap = fc.newSnapshot()) {
- Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
- Assert.assertEquals((Long) 7L, wcMap.get(snap, "dog"));
- Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
- }
-
- Map<String, Long> expectedCounts = new HashMap<>();
- expectedCounts.put("cat", 4L);
- expectedCounts.put("dog", 7L);
- expectedCounts.put("fish", 2L);
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (Transaction tx = fc.newTransaction()) {
- wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", -7L));
- tx.commit();
- }
-
- // there may be outstanding update and a current value for the key in this case
- try (Snapshot snap = fc.newSnapshot()) {
- Assert.assertEquals((Long) 5L, wcMap.get(snap, "cat"));
- Assert.assertNull(wcMap.get(snap, "dog"));
- Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
- }
-
- miniFluo.waitForObservers();
-
- try (Snapshot snap = fc.newSnapshot()) {
- Assert.assertEquals((Long) 5L, wcMap.get(snap, "cat"));
- Assert.assertNull(wcMap.get(snap, "dog"));
- Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
- }
-
- expectedCounts.put("cat", 5L);
- expectedCounts.remove("dog");
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
- }
- }
-
- @Test
- public void testBasic() {
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0001", "dog cat"));
- loader.execute(new DocumentLoader("0002", "cat hamster"));
- loader.execute(new DocumentLoader("0003", "milk bread cat food"));
- loader.execute(new DocumentLoader("0004", "zoo big cat"));
- }
-
- miniFluo.waitForObservers();
-
- try (Snapshot snap = fc.newSnapshot()) {
- Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
- Assert.assertEquals((Long) 1L, wcMap.get(snap, "milk"));
- }
-
- Map<String, Long> expectedCounts = new HashMap<>();
- expectedCounts.put("dog", 1L);
- expectedCounts.put("cat", 4L);
- expectedCounts.put("hamster", 1L);
- expectedCounts.put("milk", 1L);
- expectedCounts.put("bread", 1L);
- expectedCounts.put("food", 1L);
- expectedCounts.put("zoo", 1L);
- expectedCounts.put("big", 1L);
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0001", "dog feline"));
- }
-
- miniFluo.waitForObservers();
-
- expectedCounts.put("cat", 3L);
- expectedCounts.put("feline", 1L);
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- // swap contents of two documents... should not change doc counts
- loader.execute(new DocumentLoader("0003", "zoo big cat"));
- loader.execute(new DocumentLoader("0004", "milk bread cat food"));
- }
-
- miniFluo.waitForObservers();
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0003", "zoo big cat"));
- loader.execute(new DocumentLoader("0004", "zoo big cat"));
- }
-
- miniFluo.waitForObservers();
-
- expectedCounts.put("zoo", 2L);
- expectedCounts.put("big", 2L);
- expectedCounts.remove("milk");
- expectedCounts.remove("bread");
- expectedCounts.remove("food");
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0002", "cat cat hamster hamster"));
- }
-
- miniFluo.waitForObservers();
-
- expectedCounts.put("cat", 4L);
- expectedCounts.put("hamster", 2L);
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0002", "dog hamster"));
- }
-
- miniFluo.waitForObservers();
-
- expectedCounts.put("cat", 2L);
- expectedCounts.put("hamster", 1L);
- expectedCounts.put("dog", 2L);
-
- Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
- }
- }
-
- private static String randDocId(Random rand) {
- return String.format("%04d", rand.nextInt(5000));
- }
-
- private static String randomDocument(Random rand) {
- StringBuilder sb = new StringBuilder();
-
- String sep = "";
- for (int i = 2; i < rand.nextInt(18); i++) {
- sb.append(sep);
- sep = " ";
- sb.append(String.format("%05d", rand.nextInt(50000)));
- }
-
- return sb.toString();
- }
-
- public void diff(Map<String, Long> m1, Map<String, Long> m2) {
- for (String word : m1.keySet()) {
- Long v1 = m1.get(word);
- Long v2 = m2.get(word);
-
- if (v2 == null || !v1.equals(v2)) {
- System.out.println(word + " " + v1 + " != " + v2);
- }
- }
-
- for (String word : m2.keySet()) {
- Long v1 = m1.get(word);
- Long v2 = m2.get(word);
-
- if (v1 == null) {
- System.out.println(word + " null != " + v2);
- }
- }
- }
-
- @Test
- public void testStress() throws Exception {
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- Random rand = new Random();
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- for (int i = 0; i < 1000; i++) {
- loader.execute(new DocumentLoader(randDocId(rand), randomDocument(rand)));
- }
- }
-
- miniFluo.waitForObservers();
- assertWordCountsEqual(fc);
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- for (int i = 0; i < 100; i++) {
- loader.execute(new DocumentLoader(randDocId(rand), randomDocument(rand)));
- }
- }
-
- miniFluo.waitForObservers();
- assertWordCountsEqual(fc);
- }
- }
-
- private void assertWordCountsEqual(FluoClient fc) {
- Map<String, Long> expected = computeWordCounts(fc);
- Map<String, Long> actual = getComputedWordCounts(fc);
- if (!expected.equals(actual)) {
- diff(expected, actual);
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentLoader.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentLoader.java
deleted file mode 100644
index 3e68a3e..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentLoader.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import org.apache.fluo.recipes.types.TypedLoader;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-
-public class DocumentLoader extends TypedLoader {
-
- String docid;
- String doc;
-
- DocumentLoader(String docid, String doc) {
- this.docid = docid;
- this.doc = doc;
- }
-
- @Override
- public void load(TypedTransactionBase tx, Context context) throws Exception {
- tx.mutate().row("d:" + docid).fam("content").qual("new").set(doc);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentObserver.java
deleted file mode 100644
index 974f13b..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/DocumentObserver.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.recipes.types.TypedObserver;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-
-public class DocumentObserver extends TypedObserver {
-
- CollisionFreeMap<String, Long> wcm;
-
- @Override
- public void init(Context context) throws Exception {
- wcm = CollisionFreeMap.getInstance(CollisionFreeMapIT.MAP_ID, context.getAppConfiguration());
- }
-
- @Override
- public ObservedColumn getObservedColumn() {
- return new ObservedColumn(new Column("content", "new"), NotificationType.STRONG);
- }
-
- static Map<String, Long> getWordCounts(String doc) {
- Map<String, Long> wordCounts = new HashMap<>();
- String[] words = doc.split(" ");
- for (String word : words) {
- if (word.isEmpty()) {
- continue;
- }
- wordCounts.merge(word, 1L, Long::sum);
- }
-
- return wordCounts;
- }
-
- @Override
- public void process(TypedTransactionBase tx, Bytes row, Column col) {
- String newContent = tx.get().row(row).col(col).toString();
- String currentContent = tx.get().row(row).fam("content").qual("current").toString("");
-
- Map<String, Long> newWordCounts = getWordCounts(newContent);
- Map<String, Long> currentWordCounts = getWordCounts(currentContent);
-
- Map<String, Long> changes = calculateChanges(newWordCounts, currentWordCounts);
-
- wcm.update(tx, changes);
-
- tx.mutate().row(row).fam("content").qual("current").set(newContent);
- }
-
- private static Map<String, Long> calculateChanges(Map<String, Long> newCounts,
- Map<String, Long> currCounts) {
- Map<String, Long> changes = new HashMap<>();
-
- // guava Maps class
- MapDifference<String, Long> diffs = Maps.difference(currCounts, newCounts);
-
- // compute the diffs for words that changed
- changes.putAll(Maps.transformValues(diffs.entriesDiffering(), vDiff -> vDiff.rightValue()
- - vDiff.leftValue()));
-
- // add all new words
- changes.putAll(diffs.entriesOnlyOnRight());
-
- // subtract all words no longer present
- changes.putAll(Maps.transformValues(diffs.entriesOnlyOnLeft(), l -> l * -1));
-
- return changes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/OptionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/OptionsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/OptionsTest.java
deleted file mode 100644
index ca8dd27..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/OptionsTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.recipes.map.CollisionFreeMap.Options;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class OptionsTest {
- @Test
- public void testExportQueueOptions() {
- FluoConfiguration conf = new FluoConfiguration();
-
- CollisionFreeMap.configure(conf, new Options("Q1", "CT", "KT", "VT", 100));
- CollisionFreeMap.configure(conf, new Options("Q2", "CT2", "KT2", "VT2", 200)
- .setBucketsPerTablet(20).setBufferSize(1000000));
-
- Options opts1 = new Options("Q1", conf.getAppConfiguration());
-
- Assert.assertEquals(opts1.combinerType, "CT");
- Assert.assertEquals(opts1.keyType, "KT");
- Assert.assertEquals(opts1.valueType, "VT");
- Assert.assertEquals(opts1.numBuckets, 100);
- Assert.assertEquals(opts1.bucketsPerTablet.intValue(), Options.DEFAULT_BUCKETS_PER_TABLET);
- Assert.assertEquals(opts1.bufferSize.intValue(), Options.DEFAULT_BUFFER_SIZE);
-
- Options opts2 = new Options("Q2", conf.getAppConfiguration());
-
- Assert.assertEquals(opts2.combinerType, "CT2");
- Assert.assertEquals(opts2.keyType, "KT2");
- Assert.assertEquals(opts2.valueType, "VT2");
- Assert.assertEquals(opts2.numBuckets, 200);
- Assert.assertEquals(opts2.bucketsPerTablet.intValue(), 20);
- Assert.assertEquals(opts2.bufferSize.intValue(), 1000000);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/SplitsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/SplitsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/SplitsTest.java
deleted file mode 100644
index a81e721..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/SplitsTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import com.google.common.collect.Lists;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.common.Pirtos;
-import org.apache.fluo.recipes.map.CollisionFreeMap.Options;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class SplitsTest {
- private static List<Bytes> sort(List<Bytes> in) {
- ArrayList<Bytes> out = new ArrayList<>(in);
- Collections.sort(out);
- return out;
- }
-
- @Test
- public void testSplits() {
-
- Options opts = new Options("foo", WordCountCombiner.class, String.class, Long.class, 3);
- opts.setBucketsPerTablet(1);
- FluoConfiguration fluoConfig = new FluoConfiguration();
- CollisionFreeMap.configure(fluoConfig, opts);
-
- Pirtos pirtos1 =
- CollisionFreeMap.getTableOptimizations("foo", fluoConfig.getAppConfiguration());
- List<Bytes> expected1 =
- Lists.transform(
- Arrays.asList("foo:d:1", "foo:d:2", "foo:d:~", "foo:u:1", "foo:u:2", "foo:u:~"),
- Bytes::of);
-
- Assert.assertEquals(expected1, sort(pirtos1.getSplits()));
-
- Options opts2 = new Options("bar", WordCountCombiner.class, String.class, Long.class, 6);
- opts2.setBucketsPerTablet(2);
- CollisionFreeMap.configure(fluoConfig, opts2);
-
- Pirtos pirtos2 =
- CollisionFreeMap.getTableOptimizations("bar", fluoConfig.getAppConfiguration());
- List<Bytes> expected2 =
- Lists.transform(
- Arrays.asList("bar:d:2", "bar:d:4", "bar:d:~", "bar:u:2", "bar:u:4", "bar:u:~"),
- Bytes::of);
- Assert.assertEquals(expected2, sort(pirtos2.getSplits()));
-
- Pirtos pirtos3 = CollisionFreeMap.getTableOptimizations(fluoConfig.getAppConfiguration());
-
- ArrayList<Bytes> expected3 = new ArrayList<>(expected2);
- expected3.addAll(expected1);
-
- Assert.assertEquals(expected3, sort(pirtos3.getSplits()));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/TestSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/TestSerializer.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/TestSerializer.java
deleted file mode 100644
index 4143b3f..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/TestSerializer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-
-public class TestSerializer implements SimpleSerializer {
-
- @Override
- public <T> byte[] serialize(T obj) {
- return obj.toString().getBytes();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> T deserialize(byte[] serObj, Class<T> clazz) {
- if (clazz.equals(Long.class)) {
- return (T) Long.valueOf(new String(serObj));
- }
-
- if (clazz.equals(String.class)) {
- return (T) new String(serObj);
- }
-
- throw new IllegalArgumentException();
- }
-
- @Override
- public void init(SimpleConfiguration appConfig) {}
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountCombiner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountCombiner.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountCombiner.java
deleted file mode 100644
index 15cc040..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountCombiner.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Iterator;
-import java.util.Optional;
-
-public class WordCountCombiner implements Combiner<String, Long> {
- @Override
- public Optional<Long> combine(String key, Iterator<Long> updates) {
- long sum = 0;
-
- while (updates.hasNext()) {
- sum += updates.next();
- }
-
- if (sum == 0) {
- return Optional.empty();
- } else {
- return Optional.of(sum);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountObserver.java
deleted file mode 100644
index cd8edbf..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/WordCountObserver.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Iterator;
-import java.util.Optional;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-
-public class WordCountObserver extends UpdateObserver<String, Long> {
-
- @Override
- public void updatingValues(TransactionBase tx, Iterator<Update<String, Long>> updates) {
-
- while (updates.hasNext()) {
- Update<String, Long> update = updates.next();
-
- Optional<Long> oldVal = update.getOldValue();
- Optional<Long> newVal = update.getNewValue();
-
- if (oldVal.isPresent()) {
- String oldRow = String.format("iwc:%09d:%s", oldVal.get(), update.getKey());
- tx.delete(Bytes.of(oldRow), new Column(Bytes.EMPTY, Bytes.EMPTY));
- }
-
- if (newVal.isPresent()) {
- String newRow = String.format("iwc:%09d:%s", newVal.get(), update.getKey());
- tx.set(Bytes.of(newRow), new Column(Bytes.EMPTY, Bytes.EMPTY), Bytes.EMPTY);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/transaction/RecordingTransactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/transaction/RecordingTransactionTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/transaction/RecordingTransactionTest.java
deleted file mode 100644
index c4de7b4..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/transaction/RecordingTransactionTest.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.transaction;
-
-import java.util.AbstractMap;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.recipes.types.StringEncoder;
-import org.apache.fluo.recipes.types.TypeLayer;
-import org.apache.fluo.recipes.types.TypedTransaction;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.mock;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-
-public class RecordingTransactionTest {
-
- private Transaction tx;
- private RecordingTransaction rtx;
- private TypeLayer tl = new TypeLayer(new StringEncoder());
-
- @Before
- public void setUp() {
- tx = mock(Transaction.class);
- rtx = RecordingTransaction.wrap(tx);
- }
-
- @Test
- public void testTx() {
- rtx.set(Bytes.of("r1"), new Column("cf1"), Bytes.of("v1"));
- rtx.set(Bytes.of("r2"), new Column("cf2", "cq2"), Bytes.of("v2"));
- rtx.delete(Bytes.of("r3"), new Column("cf3"));
- expect(tx.get(Bytes.of("r4"), new Column("cf4"))).andReturn(Bytes.of("v4"));
- replay(tx);
- rtx.get(Bytes.of("r4"), new Column("cf4"));
-
- List<LogEntry> entries = rtx.getTxLog().getLogEntries();
- Assert.assertEquals(4, entries.size());
- Assert.assertEquals("LogEntry{op=SET, row=r1, col=cf1 , value=v1}", entries.get(0).toString());
- Assert.assertEquals("LogEntry{op=SET, row=r2, col=cf2 cq2 , value=v2}", entries.get(1)
- .toString());
- Assert
- .assertEquals("LogEntry{op=DELETE, row=r3, col=cf3 , value=}", entries.get(2).toString());
- Assert.assertEquals("LogEntry{op=GET, row=r4, col=cf4 , value=v4}", entries.get(3).toString());
- Assert.assertEquals("{r4 cf4 =v4}", rtx.getTxLog().getOperationMap(LogEntry.Operation.GET)
- .toString());
- Assert.assertEquals("{r2 cf2 cq2 =v2, r1 cf1 =v1}",
- rtx.getTxLog().getOperationMap(LogEntry.Operation.SET).toString());
- Assert.assertEquals("{r3 cf3 =}", rtx.getTxLog().getOperationMap(LogEntry.Operation.DELETE)
- .toString());
- }
-
- @Test
- public void testTypedTx() {
- TypedTransaction ttx = tl.wrap(rtx);
- ttx.mutate().row("r5").fam("cf5").qual("cq5").set("1");
- ttx.mutate().row("r6").fam("cf6").qual("cq6").set("1");
- List<LogEntry> entries = rtx.getTxLog().getLogEntries();
- Assert.assertEquals(2, entries.size());
- Assert.assertEquals("LogEntry{op=SET, row=r5, col=cf5 cq5 , value=1}", entries.get(0)
- .toString());
- Assert.assertEquals("LogEntry{op=SET, row=r6, col=cf6 cq6 , value=1}", entries.get(1)
- .toString());
- }
-
- @Test
- public void testFilter() {
- rtx = RecordingTransaction.wrap(tx, le -> le.getColumn().getFamily().toString().equals("cfa"));
- TypedTransaction ttx = tl.wrap(rtx);
- ttx.mutate().row("r1").fam("cfa").qual("cq1").set("1");
- ttx.mutate().row("r2").fam("cfb").qual("cq2").set("2");
- ttx.mutate().row("r3").fam("cfa").qual("cq3").set("3");
- List<LogEntry> entries = rtx.getTxLog().getLogEntries();
- Assert.assertEquals(2, entries.size());
- Assert.assertEquals("LogEntry{op=SET, row=r1, col=cfa cq1 , value=1}", entries.get(0)
- .toString());
- Assert.assertEquals("LogEntry{op=SET, row=r3, col=cfa cq3 , value=3}", entries.get(1)
- .toString());
- }
-
- @Test
- public void testClose() {
- tx.close();
- replay(tx);
- rtx.close();
- verify(tx);
- }
-
- @Test
- public void testCommit() {
- tx.commit();
- replay(tx);
- rtx.commit();
- verify(tx);
- }
-
- @Test
- public void testDelete() {
- tx.delete(Bytes.of("r"), Column.EMPTY);
- replay(tx);
- rtx.delete(Bytes.of("r"), Column.EMPTY);
- verify(tx);
- }
-
- @Test
- public void testGet() {
- expect(tx.get(Bytes.of("r"), Column.EMPTY)).andReturn(Bytes.of("v"));
- replay(tx);
- Assert.assertEquals(Bytes.of("v"), rtx.get(Bytes.of("r"), Column.EMPTY));
- verify(tx);
- }
-
- @Test
- public void testGetColumns() {
- expect(tx.get(Bytes.of("r"), Collections.emptySet())).andReturn(Collections.emptyMap());
- replay(tx);
- Assert.assertEquals(Collections.emptyMap(), rtx.get(Bytes.of("r"), Collections.emptySet()));
- verify(tx);
- }
-
- @Test
- public void testGetRows() {
- expect(tx.get(Collections.emptyList(), Collections.emptySet())).andReturn(
- Collections.emptyMap());
- replay(tx);
- Assert.assertEquals(Collections.emptyMap(),
- rtx.get(Collections.emptyList(), Collections.emptySet()));
- verify(tx);
- }
-
- @Test
- public void testGetScanNull() {
- ScannerConfiguration scanConfig = new ScannerConfiguration();
- expect(tx.get(scanConfig)).andReturn(null);
- replay(tx);
- Assert.assertNull(rtx.get(scanConfig));
- verify(tx);
- }
-
- @Test
- public void testGetScanIter() {
- ScannerConfiguration scanConfig = new ScannerConfiguration();
- expect(tx.get(scanConfig)).andReturn(new RowIterator() {
-
- private boolean hasNextRow = true;
-
- @Override
- public boolean hasNext() {
- return hasNextRow;
- }
-
- @Override
- public Map.Entry<Bytes, ColumnIterator> next() {
- hasNextRow = false;
- return new AbstractMap.SimpleEntry<>(Bytes.of("r7"), new ColumnIterator() {
-
- private boolean hasNextCol = true;
-
- @Override
- public boolean hasNext() {
- return hasNextCol;
- }
-
- @Override
- public Map.Entry<Column, Bytes> next() {
- hasNextCol = false;
- return new AbstractMap.SimpleEntry<>(new Column("cf7", "cq7"), Bytes.of("v7"));
- }
- });
- }
- });
- replay(tx);
- RowIterator rowIter = rtx.get(scanConfig);
- Assert.assertNotNull(rowIter);
- Assert.assertTrue(rtx.getTxLog().isEmpty());
- Assert.assertTrue(rowIter.hasNext());
- Map.Entry<Bytes, ColumnIterator> rowEntry = rowIter.next();
- Assert.assertFalse(rowIter.hasNext());
- Assert.assertEquals(Bytes.of("r7"), rowEntry.getKey());
- ColumnIterator colIter = rowEntry.getValue();
- Assert.assertTrue(colIter.hasNext());
- Assert.assertTrue(rtx.getTxLog().isEmpty());
- Map.Entry<Column, Bytes> colEntry = colIter.next();
- Assert.assertFalse(rtx.getTxLog().isEmpty());
- Assert.assertFalse(colIter.hasNext());
- Assert.assertEquals(new Column("cf7", "cq7"), colEntry.getKey());
- Assert.assertEquals(Bytes.of("v7"), colEntry.getValue());
- List<LogEntry> entries = rtx.getTxLog().getLogEntries();
- Assert.assertEquals(1, entries.size());
- Assert.assertEquals("LogEntry{op=GET, row=r7, col=cf7 cq7 , value=v7}", entries.get(0)
- .toString());
- verify(tx);
- }
-
- @Test
- public void testGetTimestamp() {
- expect(tx.getStartTimestamp()).andReturn(5L);
- replay(tx);
- Assert.assertEquals(5L, rtx.getStartTimestamp());
- verify(tx);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshot.java b/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshot.java
deleted file mode 100644
index 3cdc9ea..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshot.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.client.Snapshot;
-
-public class MockSnapshot extends MockSnapshotBase implements Snapshot {
-
- MockSnapshot(String... entries) {
- super(entries);
- }
-
- @Override
- public void close() {
- // no resources need to be closed
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshotBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshotBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshotBase.java
deleted file mode 100644
index 93372dc..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockSnapshotBase.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.fluo.api.client.SnapshotBase;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumn;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.core.impl.TxStringUtil;
-
-public class MockSnapshotBase implements SnapshotBase {
-
- final Map<Bytes, Map<Column, Bytes>> getData;
-
- /**
- * Initializes {@link #getData} using {@link #toRCVM(String...)}
- */
- MockSnapshotBase(String... entries) {
- getData = toRCVM(entries);
- }
-
- @Override
- public Bytes get(Bytes row, Column column) {
- Map<Column, Bytes> cols = getData.get(row);
- if (cols != null) {
- return cols.get(column);
- }
-
- return null;
- }
-
- @Override
- public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
- Map<Column, Bytes> ret = new HashMap<>();
- Map<Column, Bytes> cols = getData.get(row);
- if (cols != null) {
- for (Column column : columns) {
- Bytes val = cols.get(column);
- if (val != null) {
- ret.put(column, val);
- }
- }
- }
- return ret;
- }
-
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
-
- Map<Bytes, Map<Column, Bytes>> ret = new HashMap<>();
-
- for (Bytes row : rows) {
- Map<Column, Bytes> colMap = get(row, columns);
- if (colMap != null && colMap.size() > 0) {
- ret.put(row, colMap);
- }
- }
-
- return ret;
- }
-
- @Override
- public RowIterator get(ScannerConfiguration config) {
- throw new UnsupportedOperationException();
- }
-
- /**
- * toRCVM stands for "To Row Column Value Map". This is a convenience function that takes strings
- * of the format {@code <row>,<col fam>:<col qual>[:col vis],
- * <value>} and generates a row, column, value map.
- */
- public static Map<Bytes, Map<Column, Bytes>> toRCVM(String... entries) {
- Map<Bytes, Map<Column, Bytes>> ret = new HashMap<>();
-
- for (String entry : entries) {
- String[] rcv = entry.split(",");
- if (rcv.length != 3 && !(rcv.length == 2 && entry.trim().endsWith(","))) {
- throw new IllegalArgumentException(
- "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
- }
-
- Bytes row = Bytes.of(rcv[0]);
- String[] colFields = rcv[1].split(":");
-
- Column col;
- if (colFields.length == 3) {
- col = new Column(colFields[0], colFields[1], colFields[2]);
- } else if (colFields.length == 2) {
- col = new Column(colFields[0], colFields[1]);
- } else {
- throw new IllegalArgumentException(
- "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
- }
-
- Bytes val;
- if (rcv.length == 2) {
- val = Bytes.EMPTY;
- } else {
- val = Bytes.of(rcv[2]);
- }
-
- Map<Column, Bytes> cols = ret.get(row);
- if (cols == null) {
- cols = new HashMap<>();
- ret.put(row, cols);
- }
-
- cols.put(col, val);
- }
- return ret;
- }
-
- /**
- * toRCM stands for "To Row Column Map". This is a convenience function that takes strings of the
- * format {@code <row>,<col fam>:<col qual>[:col vis]} and generates a row, column map.
- */
- public static Map<Bytes, Set<Column>> toRCM(String... entries) {
- Map<Bytes, Set<Column>> ret = new HashMap<>();
-
- for (String entry : entries) {
- String[] rcv = entry.split(",");
- if (rcv.length != 2) {
- throw new IllegalArgumentException(
- "expected <row>,<col fam>:<col qual>[:col vis] but saw : " + entry);
- }
-
- Bytes row = Bytes.of(rcv[0]);
- String[] colFields = rcv[1].split(":");
-
- Column col;
- if (colFields.length == 3) {
- col = new Column(colFields[0], colFields[1], colFields[2]);
- } else if (colFields.length == 2) {
- col = new Column(colFields[0], colFields[1]);
- } else {
- throw new IllegalArgumentException(
- "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
- }
-
- Set<Column> cols = ret.get(row);
- if (cols == null) {
- cols = new HashSet<>();
- ret.put(row, cols);
- }
-
- cols.add(col);
- }
- return ret;
- }
-
- @Override
- public long getStartTimestamp() {
- throw new UnsupportedOperationException();
- }
-
-
- @Override
- public String gets(String row, Column column) {
- return TxStringUtil.gets(this, row, column);
- }
-
- @Override
- public Map<Column, String> gets(String row, Set<Column> columns) {
- return TxStringUtil.gets(this, row, columns);
- }
-
- @Override
- public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
- return TxStringUtil.gets(this, rows, columns);
- }
-
- @Override
- public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
- return TxStringUtil.gets(this, rowColumns);
- }
-
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransaction.java b/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransaction.java
deleted file mode 100644
index f187234..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransaction.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.exceptions.CommitException;
-
-public class MockTransaction extends MockTransactionBase implements Transaction {
-
- MockTransaction(String... entries) {
- super(entries);
- }
-
- @Override
- public void commit() throws CommitException {
- // does nothing
- }
-
- @Override
- public void close() {
- // no resources to close
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransactionBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransactionBase.java
deleted file mode 100644
index 07a95e9..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/types/MockTransactionBase.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.exceptions.AlreadySetException;
-
-/**
- * A very simple implementation of {@link TransactionBase} used for testing. All reads are serviced
- * from {@link #getData}. Updates are stored in {@link #setData}, {@link #deletes}, or
- * {@link #weakNotifications} depending on the update type.
- */
-public class MockTransactionBase extends MockSnapshotBase implements TransactionBase {
-
- final Map<Bytes, Map<Column, Bytes>> setData = new HashMap<>();
- final Map<Bytes, Set<Column>> deletes = new HashMap<>();
- final Map<Bytes, Set<Column>> weakNotifications = new HashMap<>();
-
- MockTransactionBase(String... entries) {
- super(entries);
- }
-
- @Override
- public void setWeakNotification(Bytes row, Column col) {
- Set<Column> cols = weakNotifications.get(row);
- if (cols == null) {
- cols = new HashSet<>();
- weakNotifications.put(row, cols);
- }
-
- cols.add(col);
- }
-
- @Override
- public void set(Bytes row, Column col, Bytes value) {
- Map<Column, Bytes> cols = setData.get(row);
- if (cols == null) {
- cols = new HashMap<>();
- setData.put(row, cols);
- }
-
- cols.put(col, value);
- }
-
- @Override
- public void delete(Bytes row, Column col) {
- Set<Column> cols = deletes.get(row);
- if (cols == null) {
- cols = new HashSet<>();
- deletes.put(row, cols);
- }
-
- cols.add(col);
- }
-
- @Override
- public void setWeakNotification(String row, Column col) {
- setWeakNotification(Bytes.of(row), col);
- }
-
- @Override
- public void set(String row, Column col, String value) throws AlreadySetException {
- set(Bytes.of(row), col, Bytes.of(value));
- }
-
- @Override
- public void delete(String row, Column col) {
- delete(Bytes.of(row), col);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/types/TypeLayerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/types/TypeLayerTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/types/TypeLayerTest.java
deleted file mode 100644
index 1139481..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/types/TypeLayerTest.java
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.recipes.types.TypedSnapshotBase.Value;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TypeLayerTest {
-
- @Test
- public void testColumns() throws Exception {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockTransactionBase tt =
- new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
- "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
-
- TypedTransactionBase ttx = tl.wrap(tt);
-
- Map<Column, Value> results =
- ttx.get().row("r2")
- .columns(ImmutableSet.of(new Column("cf2", "6"), new Column("cf2", "7")));
-
- Assert.assertNull(results.get(new Column("cf2", "6")).toInteger());
- Assert.assertEquals(0, results.get(new Column("cf2", "6")).toInteger(0));
- Assert.assertEquals(12, (int) results.get(new Column("cf2", "7")).toInteger());
- Assert.assertEquals(12, results.get(new Column("cf2", "7")).toInteger(0));
-
- Assert.assertEquals(1, results.size());
-
- results =
- ttx.get()
- .row("r2")
- .columns(
- ImmutableSet.of(new Column("cf2", "6"), new Column("cf2", "7"), new Column("cf2",
- "8")));
-
- Assert.assertNull(results.get(new Column("cf2", "6")).toInteger());
- Assert.assertEquals(0, results.get(new Column("cf2", "6")).toInteger(0));
- Assert.assertEquals(12, (int) results.get(new Column("cf2", "7")).toInteger());
- Assert.assertEquals(12, results.get(new Column("cf2", "7")).toInteger(0));
- Assert.assertEquals(13, (int) results.get(new Column("cf2", "8")).toInteger());
- Assert.assertEquals(13, results.get(new Column("cf2", "8")).toInteger(0));
-
- Assert.assertEquals(2, results.size());
-
- // test var args
- Map<Column, Value> results2 =
- ttx.get().row("r2")
- .columns(new Column("cf2", "6"), new Column("cf2", "7"), new Column("cf2", "8"));
- Assert.assertEquals(results, results2);
- }
-
- @Test
- public void testVis() throws Exception {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockTransactionBase tt = new MockTransactionBase("r1,cf1:cq1:A,v1", "r1,cf1:cq2:A&B,v2");
-
- TypedTransactionBase ttx = tl.wrap(tt);
-
- Assert.assertNull(ttx.get().row("r1").fam("cf1").qual("cq1").toString());
- Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A").toString());
- Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A".getBytes())
- .toString());
- Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A"))
- .toString());
- Assert.assertEquals("v1",
- ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A".getBytes())).toString());
-
- Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B").toString());
- Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B".getBytes())
- .toString());
- Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&B"))
- .toString());
- Assert.assertNull("v1",
- ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&B".getBytes()))
- .toString());
-
- Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B").toString("v3"));
- Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B".getBytes())
- .toString("v3"));
- Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&B"))
- .toString("v3"));
- Assert.assertEquals(
- "v3",
- ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&B".getBytes()))
- .toString("v3"));
-
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&B").set(3);
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&C".getBytes()).set(4);
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&D")).set(5);
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&F".getBytes())).set(7);
-
- Assert.assertEquals(MockTransactionBase.toRCVM("r1,cf1:cq1:A&B,3", "r1,cf1:cq1:A&C,4",
- "r1,cf1:cq1:A&D,5", "r1,cf1:cq1:A&F,7"), tt.setData);
- tt.setData.clear();
-
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&B").delete();
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&C".getBytes()).delete();
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&D")).delete();
- ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&F".getBytes())).delete();
-
- Assert.assertEquals(MockTransactionBase.toRCM("r1,cf1:cq1:A&B", "r1,cf1:cq1:A&C",
- "r1,cf1:cq1:A&D", "r1,cf1:cq1:A&F"), tt.deletes);
- tt.deletes.clear();
- Assert.assertEquals(0, tt.setData.size());
- Assert.assertEquals(0, tt.weakNotifications.size());
-
- }
-
- @Test
- public void testBuildColumn() {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- Assert.assertEquals(new Column("f0", "q0"), tl.bc().fam("f0".getBytes()).qual("q0".getBytes())
- .vis());
- Assert.assertEquals(new Column("f0", "q0"), tl.bc().fam("f0").qual("q0").vis());
- Assert.assertEquals(new Column("5", "7"), tl.bc().fam(5).qual(7).vis());
- Assert.assertEquals(new Column("5", "7"), tl.bc().fam(5l).qual(7l).vis());
- Assert.assertEquals(new Column("5", "7"), tl.bc().fam(Bytes.of("5")).qual(Bytes.of("7")).vis());
- Assert.assertEquals(new Column("5", "7"),
- tl.bc().fam(ByteBuffer.wrap("5".getBytes())).qual(ByteBuffer.wrap("7".getBytes())).vis());
-
- Assert.assertEquals(new Column("f0", "q0", "A&B"),
- tl.bc().fam("f0".getBytes()).qual("q0".getBytes()).vis("A&B"));
- Assert.assertEquals(new Column("f0", "q0", "A&C"),
- tl.bc().fam("f0").qual("q0").vis("A&C".getBytes()));
- Assert.assertEquals(new Column("5", "7", "A&D"), tl.bc().fam(5).qual(7).vis(Bytes.of("A&D")));
- Assert.assertEquals(new Column("5", "7", "A&D"),
- tl.bc().fam(5).qual(7).vis(ByteBuffer.wrap("A&D".getBytes())));
- }
-
- @Test
- public void testRead() throws Exception {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockSnapshot ms =
- new MockSnapshot("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
- "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20",
- "r3,cf3:cq3,28.195", "r4,cf4:cq4,true");
-
- TypedSnapshot tts = tl.wrap(ms);
-
- Assert.assertEquals("v1", tts.get().row("r1").fam("cf1").qual("cq1").toString());
- Assert.assertEquals("v1", tts.get().row("r1").fam("cf1").qual("cq1").toString("b"));
- Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual("8").toString());
- Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual("8").toString("b"));
- Assert.assertEquals("28.195", tts.get().row("r3").fam("cf3").qual("cq3").toString());
- Assert.assertEquals("28.195", tts.get().row("r3").fam("cf3").qual("cq3").toString("b"));
- Assert.assertEquals("true", tts.get().row("r4").fam("cf4").qual("cq4").toString());
- Assert.assertEquals("true", tts.get().row("r4").fam("cf4").qual("cq4").toString("b"));
-
- // try converting to different types
- Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual(8).toString());
- Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual(8).toString("b"));
- Assert.assertEquals((Integer) 13, tts.get().row("r2").fam("cf2").qual(8).toInteger());
- Assert.assertEquals(13, tts.get().row("r2").fam("cf2").qual(8).toInteger(14));
- Assert.assertEquals((Long) 13l, tts.get().row("r2").fam("cf2").qual(8).toLong());
- Assert.assertEquals(13l, tts.get().row("r2").fam("cf2").qual(8).toLong(14l));
- Assert.assertEquals("13", new String(tts.get().row("r2").fam("cf2").qual(8).toBytes()));
- Assert.assertEquals("13",
- new String(tts.get().row("r2").fam("cf2").qual(8).toBytes("14".getBytes())));
- Assert
- .assertEquals("13", new String(tts.get().row("r2").col(new Column("cf2", "8")).toBytes()));
- Assert.assertEquals("13",
- new String(tts.get().row("r2").col(new Column("cf2", "8")).toBytes("14".getBytes())));
- Assert.assertEquals("13",
- Bytes.of(tts.get().row("r2").col(new Column("cf2", "8")).toByteBuffer()).toString());
- Assert.assertEquals(
- "13",
- Bytes.of(
- tts.get().row("r2").col(new Column("cf2", "8"))
- .toByteBuffer(ByteBuffer.wrap("14".getBytes()))).toString());
-
- // test non-existent
- Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toInteger());
- Assert.assertEquals(14, tts.get().row("r2").fam("cf3").qual(8).toInteger(14));
- Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toLong());
- Assert.assertEquals(14l, tts.get().row("r2").fam("cf3").qual(8).toLong(14l));
- Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toString());
- Assert.assertEquals("14", tts.get().row("r2").fam("cf3").qual(8).toString("14"));
- Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toBytes());
- Assert.assertEquals("14",
- new String(tts.get().row("r2").fam("cf3").qual(8).toBytes("14".getBytes())));
- Assert.assertNull(tts.get().row("r2").col(new Column("cf3", "8")).toBytes());
- Assert.assertEquals("14",
- new String(tts.get().row("r2").col(new Column("cf3", "8")).toBytes("14".getBytes())));
- Assert.assertNull(tts.get().row("r2").col(new Column("cf3", "8")).toByteBuffer());
- Assert.assertEquals(
- "14",
- Bytes.of(
- tts.get().row("r2").col(new Column("cf3", "8"))
- .toByteBuffer(ByteBuffer.wrap("14".getBytes()))).toString());
-
- // test float & double
- Assert.assertEquals((Float) 28.195f, tts.get().row("r3").fam("cf3").qual("cq3").toFloat());
- Assert.assertEquals(28.195f, tts.get().row("r3").fam("cf3").qual("cq3").toFloat(39.383f), 0.0);
- Assert.assertEquals((Double) 28.195d, tts.get().row("r3").fam("cf3").qual("cq3").toDouble());
- Assert.assertEquals(28.195d, tts.get().row("r3").fam("cf3").qual("cq3").toDouble(39.383d), 0.0);
-
- // test boolean
- Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean());
- Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean());
- Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean(false));
- Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean(false));
-
- // try different types for row
- Assert.assertEquals("20", tts.get().row(13).fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row(13l).fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13".getBytes()).fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row(ByteBuffer.wrap("13".getBytes())).fam("9").qual("17")
- .toString());
-
- // try different types for cf
- Assert.assertEquals("20", tts.get().row("13").fam(9).qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam(9l).qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9".getBytes()).qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam(ByteBuffer.wrap("9".getBytes())).qual("17")
- .toString());
-
- // try different types for cq
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual(17l).toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual(17).toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17".getBytes()).toString());
- Assert.assertEquals("20", tts.get().row("13").fam("9").qual(ByteBuffer.wrap("17".getBytes()))
- .toString());
-
- ms.close();
- tts.close();
- }
-
- @Test
- public void testWrite() throws Exception {
-
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockTransactionBase tt =
- new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
- "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
-
- TypedTransactionBase ttx = tl.wrap(tt);
-
- // test increments data
- ttx.mutate().row("13").fam("9").qual("17").increment(1);
- ttx.mutate().row("13").fam("9").qual(18).increment(2);
- ttx.mutate().row("13").fam("9").qual(19l).increment(3);
- ttx.mutate().row("13").fam("9").qual("20".getBytes()).increment(4);
- ttx.mutate().row("13").fam("9").qual(Bytes.of("21")).increment(5); // increment non existent
- ttx.mutate().row("13").col(new Column("9", "22")).increment(6); // increment non existent
- ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("23".getBytes())).increment(7); // increment
- // non
- // existent
-
- Assert.assertEquals(MockTransactionBase.toRCVM("13,9:17,21", "13,9:18,22", "13,9:19,23",
- "13,9:20,24", "13,9:21,5", "13,9:22,6", "13,9:23,7"), tt.setData);
- tt.setData.clear();
-
- // test increments long
- ttx.mutate().row("13").fam("9").qual("17").increment(1l);
- ttx.mutate().row("13").fam("9").qual(18).increment(2l);
- ttx.mutate().row("13").fam("9").qual(19l).increment(3l);
- ttx.mutate().row("13").fam("9").qual("20".getBytes()).increment(4l);
- ttx.mutate().row("13").fam("9").qual(Bytes.of("21")).increment(5l); // increment non existent
- ttx.mutate().row("13").col(new Column("9", "22")).increment(6l); // increment non existent
- ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("23".getBytes())).increment(7l); // increment
- // non
- // existent
-
- Assert.assertEquals(MockTransactionBase.toRCVM("13,9:17,21", "13,9:18,22", "13,9:19,23",
- "13,9:20,24", "13,9:21,5", "13,9:22,6", "13,9:23,7"), tt.setData);
- tt.setData.clear();
-
- // test setting data
- ttx.mutate().row("13").fam("9").qual("16").set();
- ttx.mutate().row("13").fam("9").qual("17").set(3);
- ttx.mutate().row("13").fam("9").qual(18).set(4l);
- ttx.mutate().row("13").fam("9").qual(19l).set("5");
- ttx.mutate().row("13").fam("9").qual("20".getBytes()).set("6".getBytes());
- ttx.mutate().row("13").col(new Column("9", "21")).set("7".getBytes());
- ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes()))
- .set(ByteBuffer.wrap("8".getBytes()));
- ttx.mutate().row("13").fam("9").qual("23").set(2.54f);
- ttx.mutate().row("13").fam("9").qual("24").set(-6.135d);
- ttx.mutate().row("13").fam("9").qual("25").set(false);
-
- Assert.assertEquals(MockTransactionBase.toRCVM("13,9:16,", "13,9:17,3", "13,9:18,4",
- "13,9:19,5", "13,9:20,6", "13,9:21,7", "13,9:22,8", "13,9:23,2.54", "13,9:24,-6.135",
- "13,9:25,false"), tt.setData);
- tt.setData.clear();
-
- // test deleting data
- ttx.mutate().row("13").fam("9").qual("17").delete();
- ttx.mutate().row("13").fam("9").qual(18).delete();
- ttx.mutate().row("13").fam("9").qual(19l).delete();
- ttx.mutate().row("13").fam("9").qual("20".getBytes()).delete();
- ttx.mutate().row("13").col(new Column("9", "21")).delete();
- ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes())).delete();
-
- Assert
- .assertEquals(MockTransactionBase.toRCM("13,9:17", "13,9:18", "13,9:19", "13,9:20",
- "13,9:21", "13,9:22"), tt.deletes);
- tt.deletes.clear();
- Assert.assertEquals(0, tt.setData.size());
- Assert.assertEquals(0, tt.weakNotifications.size());
-
- // test weak notifications
- ttx.mutate().row("13").fam("9").qual("17").weaklyNotify();
- ttx.mutate().row("13").fam("9").qual(18).weaklyNotify();
- ttx.mutate().row("13").fam("9").qual(19l).weaklyNotify();
- ttx.mutate().row("13").fam("9").qual("20".getBytes()).weaklyNotify();
- ttx.mutate().row("13").col(new Column("9", "21")).weaklyNotify();
- ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes())).weaklyNotify();
-
- Assert
- .assertEquals(MockTransactionBase.toRCM("13,9:17", "13,9:18", "13,9:19", "13,9:20",
- "13,9:21", "13,9:22"), tt.weakNotifications);
- tt.weakNotifications.clear();
- Assert.assertEquals(0, tt.setData.size());
- Assert.assertEquals(0, tt.deletes.size());
- }
-
- @Test
- public void testMultiRow() throws Exception {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockTransactionBase tt =
- new MockTransactionBase("11,cf1:cq1,1", "11,cf1:cq2,2", "12,cf1:cq1,3", "12,cf1:cq2,4",
- "13,cf1:cq1,5", "13,cf1:cq2,6");
-
- TypedTransactionBase ttx = tl.wrap(tt);
-
- Bytes br1 = Bytes.of("11");
- Bytes br2 = Bytes.of("12");
- Bytes br3 = Bytes.of("13");
-
- Column c1 = new Column("cf1", "cq1");
- Column c2 = new Column("cf1", "cq2");
-
- Map<Bytes, Map<Column, Value>> map1 =
- ttx.get().rows(Arrays.asList(br1, br2)).columns(c1).toBytesMap();
-
- Assert.assertEquals(map1, ttx.get().rows(br1, br2).columns(c1).toBytesMap());
-
- Assert.assertEquals("1", map1.get(br1).get(c1).toString());
- Assert.assertEquals("1", map1.get(br1).get(c1).toString("5"));
- Assert.assertEquals((Long) (1l), map1.get(br1).get(c1).toLong());
- Assert.assertEquals(1l, map1.get(br1).get(c1).toLong(5));
- Assert.assertEquals((Integer) (1), map1.get(br1).get(c1).toInteger());
- Assert.assertEquals(1, map1.get(br1).get(c1).toInteger(5));
-
- Assert.assertEquals("5", map1.get(br3).get(c1).toString("5"));
- Assert.assertNull(map1.get(br3).get(c1).toString());
- Assert.assertEquals(5l, map1.get(br3).get(c1).toLong(5l));
- Assert.assertNull(map1.get(br3).get(c1).toLong());
- Assert.assertEquals(5, map1.get(br1).get(c2).toInteger(5));
- Assert.assertNull(map1.get(br1).get(c2).toInteger());
-
- Assert.assertEquals(2, map1.size());
- Assert.assertEquals(1, map1.get(br1).size());
- Assert.assertEquals(1, map1.get(br2).size());
- Assert.assertEquals("3", map1.get(br2).get(c1).toString());
-
- Map<String, Map<Column, Value>> map2 =
- ttx.get().rowsString(Arrays.asList("11", "13")).columns(c1).toStringMap();
-
- Assert.assertEquals(map2, ttx.get().rowsString("11", "13").columns(c1).toStringMap());
-
- Assert.assertEquals(2, map2.size());
- Assert.assertEquals(1, map2.get("11").size());
- Assert.assertEquals(1, map2.get("13").size());
- Assert.assertEquals((Long) (1l), map2.get("11").get(c1).toLong());
- Assert.assertEquals(5l, map2.get("13").get(c1).toLong(6));
-
- Map<Long, Map<Column, Value>> map3 =
- ttx.get().rowsLong(Arrays.asList(11l, 13l)).columns(c1).toLongMap();
-
- Assert.assertEquals(map3, ttx.get().rowsLong(11l, 13l).columns(c1).toLongMap());
-
- Assert.assertEquals(2, map3.size());
- Assert.assertEquals(1, map3.get(11l).size());
- Assert.assertEquals(1, map3.get(13l).size());
- Assert.assertEquals((Long) (1l), map3.get(11l).get(c1).toLong());
- Assert.assertEquals(5l, map3.get(13l).get(c1).toLong(6));
-
- Map<Integer, Map<Column, Value>> map4 =
- ttx.get().rowsInteger(Arrays.asList(11, 13)).columns(c1).toIntegerMap();
-
- Assert.assertEquals(map4, ttx.get().rowsInteger(11, 13).columns(c1).toIntegerMap());
-
- Assert.assertEquals(2, map4.size());
- Assert.assertEquals(1, map4.get(11).size());
- Assert.assertEquals(1, map4.get(13).size());
- Assert.assertEquals((Long) (1l), map4.get(11).get(c1).toLong());
- Assert.assertEquals(5l, map4.get(13).get(c1).toLong(6));
-
- Map<Integer, Map<Column, Value>> map5 =
- ttx.get().rowsBytes(Arrays.asList("11".getBytes(), "13".getBytes())).columns(c1)
- .toIntegerMap();
-
- Assert.assertEquals(map5, ttx.get().rowsBytes("11".getBytes(), "13".getBytes()).columns(c1)
- .toIntegerMap());
-
- Assert.assertEquals(2, map5.size());
- Assert.assertEquals(1, map5.get(11).size());
- Assert.assertEquals(1, map5.get(13).size());
- Assert.assertEquals((Long) (1l), map5.get(11).get(c1).toLong());
- Assert.assertEquals(5l, map5.get(13).get(c1).toLong(6));
-
- Map<Integer, Map<Column, Value>> map6 =
- ttx.get()
- .rowsByteBuffers(
- Arrays.asList(ByteBuffer.wrap("11".getBytes()), ByteBuffer.wrap("13".getBytes())))
- .columns(c1).toIntegerMap();
-
- Assert.assertEquals(
- map6,
- ttx.get()
- .rowsByteBuffers(ByteBuffer.wrap("11".getBytes()), ByteBuffer.wrap("13".getBytes()))
- .columns(c1).toIntegerMap());
-
- Assert.assertEquals(2, map6.size());
- Assert.assertEquals(1, map6.get(11).size());
- Assert.assertEquals(1, map6.get(13).size());
- Assert.assertEquals((Long) (1l), map6.get(11).get(c1).toLong());
- Assert.assertEquals(5l, map6.get(13).get(c1).toLong(6));
-
- }
-
- @Test
- public void testBasic() throws Exception {
- TypeLayer tl = new TypeLayer(new StringEncoder());
-
- MockTransactionBase tt =
- new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
- "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
-
- TypedTransactionBase ttx = tl.wrap(tt);
-
- Assert.assertEquals(Bytes.of("12"), ttx.get(Bytes.of("r2"), new Column("cf2", "7")));
- Assert.assertNull(ttx.get(Bytes.of("r2"), new Column("cf2", "9")));
-
- Map<Column, Bytes> map =
- ttx.get(Bytes.of("r2"), ImmutableSet.of(new Column("cf2", "7"), new Column("cf2", "8")));
- Assert.assertEquals(2, map.size());
- Assert.assertEquals("12", map.get(new Column("cf2", "7")).toString());
- Assert.assertEquals("13", map.get(new Column("cf2", "8")).toString());
-
- map = ttx.get(Bytes.of("r6"), ImmutableSet.of(new Column("cf2", "7"), new Column("cf2", "8")));
- Assert.assertEquals(0, map.size());
-
- ttx.set(Bytes.of("r6"), new Column("cf2", "7"), Bytes.of("3"));
- Assert.assertEquals(MockTransactionBase.toRCVM("r6,cf2:7,3"), tt.setData);
- tt.setData.clear();
-
- Map<Bytes, Map<Column, Bytes>> map2 =
- ttx.get(ImmutableSet.of(Bytes.of("r1"), Bytes.of("r2")),
- ImmutableSet.of(new Column("cf1", "cq1"), new Column("cf2", "8")));
- Assert.assertEquals(MockTransactionBase.toRCVM("r1,cf1:cq1,v1", "r2,cf2:8,13"), map2);
-
- ttx.delete(Bytes.of("r6"), new Column("cf2", "7"));
- Assert.assertEquals(MockTransactionBase.toRCM("r6,cf2:7"), tt.deletes);
- tt.deletes.clear();
-
- ttx.setWeakNotification(Bytes.of("r6"), new Column("cf2", "8"));
- Assert.assertEquals(MockTransactionBase.toRCM("r6,cf2:8"), tt.weakNotifications);
- tt.weakNotifications.clear();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
----------------------------------------------------------------------
diff --git a/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java b/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
index 1ade68f..851f045 100644
--- a/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
+++ b/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
@@ -30,7 +30,7 @@ import com.esotericsoftware.kryo.pool.KryoPool;
import com.google.common.base.Preconditions;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
public class KryoSimplerSerializer implements SimpleSerializer, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/kryo/src/test/java/org/apache/fluo/recipes/core/serialization/KryoSimpleSerializerTest.java
----------------------------------------------------------------------
diff --git a/modules/kryo/src/test/java/org/apache/fluo/recipes/core/serialization/KryoSimpleSerializerTest.java b/modules/kryo/src/test/java/org/apache/fluo/recipes/core/serialization/KryoSimpleSerializerTest.java
new file mode 100644
index 0000000..95a26a9
--- /dev/null
+++ b/modules/kryo/src/test/java/org/apache/fluo/recipes/core/serialization/KryoSimpleSerializerTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.serialization;
+
+import com.esotericsoftware.kryo.pool.KryoFactory;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.recipes.kryo.KryoSimplerSerializer;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class KryoSimpleSerializerTest {
+
+ private static final KryoFactory KRYO_FACTORY = new KryoSimplerSerializer.DefaultFactory();
+
+ public void testColumn() {
+ SimpleSerializer serializer = new KryoSimplerSerializer(KRYO_FACTORY);
+ Column before = new Column("a", "b");
+ byte[] barray = serializer.serialize(before);
+ Column after = serializer.deserialize(barray, Column.class);
+ Assert.assertEquals(before, after);
+ }
+
+ @Test
+ public void testBytes() {
+ SimpleSerializer serializer = new KryoSimplerSerializer(KRYO_FACTORY);
+ Bytes before = Bytes.of("test");
+ byte[] barray = serializer.serialize(before);
+ Bytes after = serializer.deserialize(barray, Bytes.class);
+ Assert.assertEquals(before, after);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/kryo/src/test/java/org/apache/fluo/recipes/serialization/KryoSimpleSerializerTest.java
----------------------------------------------------------------------
diff --git a/modules/kryo/src/test/java/org/apache/fluo/recipes/serialization/KryoSimpleSerializerTest.java b/modules/kryo/src/test/java/org/apache/fluo/recipes/serialization/KryoSimpleSerializerTest.java
deleted file mode 100644
index 9e4ca4b..0000000
--- a/modules/kryo/src/test/java/org/apache/fluo/recipes/serialization/KryoSimpleSerializerTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.serialization;
-
-import com.esotericsoftware.kryo.pool.KryoFactory;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.recipes.kryo.KryoSimplerSerializer;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class KryoSimpleSerializerTest {
-
- private static final KryoFactory KRYO_FACTORY = new KryoSimplerSerializer.DefaultFactory();
-
- public void testColumn() {
- SimpleSerializer serializer = new KryoSimplerSerializer(KRYO_FACTORY);
- Column before = new Column("a", "b");
- byte[] barray = serializer.serialize(before);
- Column after = serializer.deserialize(barray, Column.class);
- Assert.assertEquals(before, after);
- }
-
- @Test
- public void testBytes() {
- SimpleSerializer serializer = new KryoSimplerSerializer(KRYO_FACTORY);
- Bytes before = Bytes.of("test");
- byte[] barray = serializer.serialize(before);
- Bytes after = serializer.deserialize(barray, Bytes.class);
- Assert.assertEquals(before, after);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
----------------------------------------------------------------------
diff --git a/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java b/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
index b640751..2d1bbb4 100644
--- a/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
+++ b/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
@@ -31,7 +31,7 @@ import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.mini.MiniFluo;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
-import org.apache.fluo.recipes.common.Pirtos;
+import org.apache.fluo.recipes.core.common.Pirtos;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloExporterIT.java
----------------------------------------------------------------------
diff --git a/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloExporterIT.java b/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloExporterIT.java
index b8e282b..38986df 100644
--- a/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloExporterIT.java
+++ b/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloExporterIT.java
@@ -33,7 +33,7 @@ import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.mini.MiniFluo;
import org.apache.fluo.recipes.accumulo.export.AccumuloExporter;
import org.apache.fluo.recipes.accumulo.export.TableInfo;
-import org.apache.fluo.recipes.export.ExportQueue;
+import org.apache.fluo.recipes.core.export.ExportQueue;
import org.apache.fluo.recipes.test.AccumuloExportITBase;
import org.apache.hadoop.io.Text;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloReplicatorIT.java
----------------------------------------------------------------------
diff --git a/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloReplicatorIT.java b/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloReplicatorIT.java
index a34f21c..432fa4d 100644
--- a/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloReplicatorIT.java
+++ b/modules/test/src/test/java/org/apache/fluo/recipes/test/export/AccumuloReplicatorIT.java
@@ -33,12 +33,12 @@ import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
import org.apache.fluo.recipes.accumulo.export.AccumuloExporter;
import org.apache.fluo.recipes.accumulo.export.ReplicationExport;
import org.apache.fluo.recipes.accumulo.export.TableInfo;
-import org.apache.fluo.recipes.export.ExportQueue;
+import org.apache.fluo.recipes.core.export.ExportQueue;
import org.apache.fluo.recipes.test.AccumuloExportITBase;
-import org.apache.fluo.recipes.transaction.RecordingTransaction;
-import org.apache.fluo.recipes.types.StringEncoder;
-import org.apache.fluo.recipes.types.TypeLayer;
-import org.apache.fluo.recipes.types.TypedTransaction;
+import org.apache.fluo.recipes.core.transaction.RecordingTransaction;
+import org.apache.fluo.recipes.core.types.StringEncoder;
+import org.apache.fluo.recipes.core.types.TypeLayer;
+import org.apache.fluo.recipes.core.types.TypedTransaction;
import org.junit.Assert;
import org.junit.Test;
[03/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
new file mode 100644
index 0000000..5a1f5fe
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.common;
+
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.recipes.core.export.ExportQueue;
+import org.apache.fluo.recipes.core.map.CollisionFreeMap;
+import org.apache.fluo.recipes.core.map.CollisionFreeMap.Options;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestGrouping {
+ @Test
+ public void testTabletGrouping() {
+ FluoConfiguration conf = new FluoConfiguration();
+
+ CollisionFreeMap.configure(conf, new Options("m1", "ct", "kt", "vt", 119));
+ CollisionFreeMap.configure(conf, new Options("m2", "ct", "kt", "vt", 3));
+
+ ExportQueue.configure(conf, new ExportQueue.Options("eq1", "et", "kt", "vt", 7));
+ ExportQueue.configure(conf, new ExportQueue.Options("eq2", "et", "kt", "vt", 3));
+
+ Pirtos pirtos = CollisionFreeMap.getTableOptimizations(conf.getAppConfiguration());
+ pirtos.merge(ExportQueue.getTableOptimizations(conf.getAppConfiguration()));
+
+ Pattern pattern = Pattern.compile(pirtos.getTabletGroupingRegex());
+
+ Assert.assertEquals("m1:u:", group(pattern, "m1:u:f0c"));
+ Assert.assertEquals("m1:d:", group(pattern, "m1:d:f0c"));
+ Assert.assertEquals("m2:u:", group(pattern, "m2:u:abc"));
+ Assert.assertEquals("m2:d:", group(pattern, "m2:d:590"));
+ Assert.assertEquals("none", group(pattern, "m3:d:590"));
+
+ Assert.assertEquals("eq1:", group(pattern, "eq1:f0c"));
+ Assert.assertEquals("eq2:", group(pattern, "eq2:f0c"));
+ Assert.assertEquals("none", group(pattern, "eq3:f0c"));
+
+ // validate the assumptions this test is making
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1#")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2#")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1:~")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2:~")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:u:~")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:d:~")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:u:~")));
+ Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:d:~")));
+
+ Set<String> expectedGroups =
+ ImmutableSet.of("m1:u:", "m1:d:", "m2:u:", "m2:d:", "eq1:", "eq2:");
+
+ // ensure all splits group as expected
+ for (Bytes split : pirtos.getSplits()) {
+ String g = group(pattern, split.toString());
+
+ if (expectedGroups.contains(g)) {
+ Assert.assertTrue(split.toString().startsWith(g));
+ } else {
+ Assert.assertEquals("none", g);
+ Assert.assertTrue(split.toString().equals("eq1#") || split.toString().equals("eq2#"));
+ }
+
+ }
+
+ }
+
+ private String group(Pattern pattern, String endRow) {
+ Matcher m = pattern.matcher(endRow);
+ if (m.matches() && m.groupCount() == 1) {
+ return m.group(1);
+ }
+ return "none";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TransientRegistryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TransientRegistryTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TransientRegistryTest.java
new file mode 100644
index 0000000..a1be5c9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TransientRegistryTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.common;
+
+import java.util.HashSet;
+
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TransientRegistryTest {
+ @Test
+ public void testBasic() {
+ FluoConfiguration fluoConfig = new FluoConfiguration();
+
+ HashSet<RowRange> expected = new HashSet<>();
+
+ TransientRegistry tr = new TransientRegistry(fluoConfig.getAppConfiguration());
+
+ RowRange rr1 = new RowRange(Bytes.of("pr1:g"), Bytes.of("pr1:q"));
+ tr.addTransientRange("foo", rr1);
+ expected.add(rr1);
+
+ tr = new TransientRegistry(fluoConfig.getAppConfiguration());
+ Assert.assertEquals(expected, new HashSet<>(tr.getTransientRanges()));
+
+ RowRange rr2 = new RowRange(Bytes.of("pr2:j"), Bytes.of("pr2:m"));
+ tr.addTransientRange("bar", rr2);
+ expected.add(rr2);
+
+ tr = new TransientRegistry(fluoConfig.getAppConfiguration());
+ Assert.assertEquals(expected, new HashSet<>(tr.getTransientRanges()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/data/RowHasherTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/data/RowHasherTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/data/RowHasherTest.java
new file mode 100644
index 0000000..9250d73
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/data/RowHasherTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.data;
+
+import org.apache.fluo.api.data.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class RowHasherTest {
+
+ @Test
+ public void testBadPrefixes() {
+ String[] badPrefixes =
+ {"q:she6:test1", "q:she6:test1", "p:Mhe6:test1", "p;she6:test1", "p:she6;test1",
+ "p;she6;test1", "p:+he6:test1", "p:s?e6:test1", "p:sh{6:test1", "p:sh6:"};
+
+ RowHasher rh = new RowHasher("p");
+ for (String badPrefix : badPrefixes) {
+ try {
+ rh.removeHash(Bytes.of(badPrefix));
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ }
+ }
+ }
+
+ @Test
+ public void testBasic() {
+ RowHasher rh = new RowHasher("p");
+ Assert.assertTrue(rh.removeHash(rh.addHash("abc")).toString().equals("abc"));
+ rh = new RowHasher("p2");
+ Assert.assertTrue(rh.removeHash(rh.addHash("abc")).toString().equals("abc"));
+
+ Assert.assertTrue(rh.addHash("abc").toString().startsWith("p2:"));
+
+ // test to ensure hash is stable over time
+ Assert.assertEquals("p2:she6:test1", rh.addHash("test1").toString());
+ Assert.assertEquals("p2:hgt0:0123456789abcdefghijklmnopqrstuvwxyz",
+ rh.addHash("0123456789abcdefghijklmnopqrstuvwxyz").toString());
+ Assert.assertEquals("p2:fluo:86ce3b094982c6a", rh.addHash("86ce3b094982c6a").toString());
+ }
+
+ @Test
+ public void testBalancerRegex() {
+ RowHasher rh = new RowHasher("p");
+ String regex = rh.getTableOptimizations(3).getTabletGroupingRegex();
+ Assert.assertEquals("(\\Qp:\\E).*", regex);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentLoader.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentLoader.java
new file mode 100644
index 0000000..5e7f224
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentLoader.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.fluo.recipes.core.types.TypedLoader;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+
+public class DocumentLoader extends TypedLoader {
+
+ String docid;
+ String refs[];
+
+ DocumentLoader(String docid, String... refs) {
+ this.docid = docid;
+ this.refs = refs;
+ }
+
+ @Override
+ public void load(TypedTransactionBase tx, Context context) throws Exception {
+ tx.mutate().row("d:" + docid).fam("content").qual("new").set(StringUtils.join(refs, " "));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentObserver.java
new file mode 100644
index 0000000..c4c11d8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/DocumentObserver.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.recipes.core.export.ExportTestBase.RefExporter;
+import org.apache.fluo.recipes.core.types.TypedObserver;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+
+public class DocumentObserver extends TypedObserver {
+
+ ExportQueue<String, RefUpdates> refExportQueue;
+
+ @Override
+ public void init(Context context) throws Exception {
+ refExportQueue = ExportQueue.getInstance(RefExporter.QUEUE_ID, context.getAppConfiguration());
+ }
+
+ @Override
+ public ObservedColumn getObservedColumn() {
+ return new ObservedColumn(new Column("content", "new"), NotificationType.STRONG);
+ }
+
+ @Override
+ public void process(TypedTransactionBase tx, Bytes row, Column col) {
+ String newContent = tx.get().row(row).col(col).toString();
+ Set<String> newRefs = new HashSet<>(Arrays.asList(newContent.split(" ")));
+ Set<String> currentRefs =
+ new HashSet<>(Arrays.asList(tx.get().row(row).fam("content").qual("current").toString("")
+ .split(" ")));
+
+ Set<String> addedRefs = new HashSet<>(newRefs);
+ addedRefs.removeAll(currentRefs);
+
+ Set<String> deletedRefs = new HashSet<>(currentRefs);
+ deletedRefs.removeAll(newRefs);
+
+ String key = row.toString().substring(2);
+ RefUpdates val = new RefUpdates(addedRefs, deletedRefs);
+
+ refExportQueue.add(tx, key, val);
+
+ tx.mutate().row(row).fam("content").qual("current").set(newContent);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportBufferIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportBufferIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportBufferIT.java
new file mode 100644
index 0000000..d54982b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportBufferIT.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.client.Transaction;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExportBufferIT extends ExportTestBase {
+
+ @Override
+ protected int getNumBuckets() {
+ return 2;
+ }
+
+ @Override
+ protected Integer getBufferSize() {
+ return 1024;
+ }
+
+ @Test
+ public void testSmallExportBuffer() {
+ // try setting the export buffer size small. Make sure everything is exported.
+
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ ExportQueue<String, RefUpdates> refExportQueue =
+ ExportQueue.getInstance(RefExporter.QUEUE_ID, fc.getAppConfiguration());
+ try (Transaction tx = fc.newTransaction()) {
+ for (int i = 0; i < 1000; i++) {
+ refExportQueue.add(tx, nk(i), new RefUpdates(ns(i + 10, i + 20), ns(new int[0])));
+ }
+
+ tx.commit();
+ }
+ }
+
+ miniFluo.waitForObservers();
+
+ Map<String, Set<String>> erefs = getExportedReferees();
+ Map<String, Set<String>> expected = new HashMap<>();
+
+ for (int i = 0; i < 1000; i++) {
+ expected.computeIfAbsent(nk(i + 10), s -> new HashSet<>()).add(nk(i));
+ expected.computeIfAbsent(nk(i + 20), s -> new HashSet<>()).add(nk(i));
+ }
+
+ assertEquals(expected, erefs);
+ int prevNumExportCalls = getNumExportCalls();
+ Assert.assertTrue(prevNumExportCalls > 10); // with small buffer there should be lots of exports
+ // calls
+
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ ExportQueue<String, RefUpdates> refExportQueue =
+ ExportQueue.getInstance(RefExporter.QUEUE_ID, fc.getAppConfiguration());
+ try (Transaction tx = fc.newTransaction()) {
+ for (int i = 0; i < 1000; i++) {
+ refExportQueue.add(tx, nk(i), new RefUpdates(ns(i + 12), ns(i + 10)));
+ }
+
+ tx.commit();
+ }
+ }
+
+ miniFluo.waitForObservers();
+
+ erefs = getExportedReferees();
+ expected = new HashMap<>();
+
+ for (int i = 0; i < 1000; i++) {
+ expected.computeIfAbsent(nk(i + 12), s -> new HashSet<>()).add(nk(i));
+ expected.computeIfAbsent(nk(i + 20), s -> new HashSet<>()).add(nk(i));
+ }
+
+ assertEquals(expected, erefs);
+ prevNumExportCalls = getNumExportCalls() - prevNumExportCalls;
+ Assert.assertTrue(prevNumExportCalls > 10);
+ }
+
+ public void assertEquals(Map<String, Set<String>> expected, Map<String, Set<String>> actual) {
+ if (!expected.equals(actual)) {
+ System.out.println("*** diff ***");
+ diff(expected, actual);
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportQueueIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportQueueIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportQueueIT.java
new file mode 100644
index 0000000..b4e167c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportQueueIT.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.client.LoaderExecutor;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExportQueueIT extends ExportTestBase {
+
+ @Test
+ public void testExport() {
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0999", "0005", "0002"));
+ loader.execute(new DocumentLoader("0002", "0999", "0042"));
+ loader.execute(new DocumentLoader("0005", "0999", "0042"));
+ loader.execute(new DocumentLoader("0042", "0999"));
+ }
+
+ miniFluo.waitForObservers();
+
+ Assert.assertEquals(ns("0002", "0005", "0042"), getExportedReferees("0999"));
+ Assert.assertEquals(ns("0999"), getExportedReferees("0002"));
+ Assert.assertEquals(ns("0999"), getExportedReferees("0005"));
+ Assert.assertEquals(ns("0002", "0005"), getExportedReferees("0042"));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0999", "0005", "0042"));
+ }
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0999", "0005"));
+ }
+
+ miniFluo.waitForObservers();
+
+ Assert.assertEquals(ns("0002", "0005", "0042"), getExportedReferees("0999"));
+ Assert.assertEquals(ns(new String[0]), getExportedReferees("0002"));
+ Assert.assertEquals(ns("0999"), getExportedReferees("0005"));
+ Assert.assertEquals(ns("0002", "0005"), getExportedReferees("0042"));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0042", "0999", "0002", "0005"));
+ loader.execute(new DocumentLoader("0005", "0002"));
+ }
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0005", "0003"));
+ }
+
+ miniFluo.waitForObservers();
+
+ Assert.assertEquals(ns("0002", "0042"), getExportedReferees("0999"));
+ Assert.assertEquals(ns("0042"), getExportedReferees("0002"));
+ Assert.assertEquals(ns("0005"), getExportedReferees("0003"));
+ Assert.assertEquals(ns("0999", "0042"), getExportedReferees("0005"));
+ Assert.assertEquals(ns("0002"), getExportedReferees("0042"));
+
+ }
+ }
+
+ @Test
+ public void exportStressTest() {
+ FluoConfiguration config = new FluoConfiguration(miniFluo.getClientConfiguration());
+ config.setLoaderQueueSize(100);
+ config.setLoaderThreads(20);
+
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+
+ loadRandom(fc, 1000, 500);
+
+ miniFluo.waitForObservers();
+
+ diff(getFluoReferees(fc), getExportedReferees());
+
+ assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
+
+ loadRandom(fc, 1000, 500);
+
+ miniFluo.waitForObservers();
+
+ assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
+
+ loadRandom(fc, 1000, 10000);
+
+ miniFluo.waitForObservers();
+
+ assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
+
+ loadRandom(fc, 1000, 10000);
+
+ miniFluo.waitForObservers();
+
+ assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportTestBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportTestBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportTestBase.java
new file mode 100644
index 0000000..c1cf3ce
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/ExportTestBase.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import com.google.common.collect.Iterators;
+import org.apache.commons.io.FileUtils;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.client.LoaderExecutor;
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.ObserverConfiguration;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.api.mini.MiniFluo;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+public class ExportTestBase {
+
+ private static Map<String, Map<String, RefInfo>> globalExports = new HashMap<>();
+ private static int exportCalls = 0;
+
+ protected static Set<String> getExportedReferees(String node) {
+ synchronized (globalExports) {
+ Set<String> ret = new HashSet<>();
+
+ Map<String, RefInfo> referees = globalExports.get(node);
+
+ if (referees == null) {
+ return ret;
+ }
+
+ referees.forEach((k, v) -> {
+ if (!v.deleted)
+ ret.add(k);
+ });
+
+ return ret;
+ }
+ }
+
+ protected static Map<String, Set<String>> getExportedReferees() {
+ synchronized (globalExports) {
+
+ Map<String, Set<String>> ret = new HashMap<>();
+
+ for (String k : globalExports.keySet()) {
+ Set<String> referees = getExportedReferees(k);
+ if (referees.size() > 0) {
+ ret.put(k, referees);
+ }
+ }
+
+ return ret;
+ }
+ }
+
+ protected static int getNumExportCalls() {
+ synchronized (globalExports) {
+ return exportCalls;
+ }
+ }
+
+ public static class RefExporter extends Exporter<String, RefUpdates> {
+
+ public static final String QUEUE_ID = "req";
+
+ private void updateExports(String key, long seq, String addedRef, boolean deleted) {
+ Map<String, RefInfo> referees = globalExports.computeIfAbsent(addedRef, k -> new HashMap<>());
+ referees.compute(key, (k, v) -> (v == null || v.seq < seq) ? new RefInfo(seq, deleted) : v);
+ }
+
+ @Override
+ protected void processExports(Iterator<SequencedExport<String, RefUpdates>> exportIterator) {
+ ArrayList<SequencedExport<String, RefUpdates>> exportList = new ArrayList<>();
+ Iterators.addAll(exportList, exportIterator);
+
+ synchronized (globalExports) {
+ exportCalls++;
+
+ for (SequencedExport<String, RefUpdates> se : exportList) {
+ for (String addedRef : se.getValue().getAddedRefs()) {
+ updateExports(se.getKey(), se.getSequence(), addedRef, false);
+ }
+
+ for (String deletedRef : se.getValue().getDeletedRefs()) {
+ updateExports(se.getKey(), se.getSequence(), deletedRef, true);
+ }
+ }
+ }
+ }
+ }
+
+ protected MiniFluo miniFluo;
+
+ protected int getNumBuckets() {
+ return 13;
+ }
+
+ protected Integer getBufferSize() {
+ return null;
+ }
+
+ @Before
+ public void setUpFluo() throws Exception {
+ FileUtils.deleteQuietly(new File("target/mini"));
+
+ FluoConfiguration props = new FluoConfiguration();
+ props.setApplicationName("eqt");
+ props.setWorkerThreads(20);
+ props.setMiniDataDir("target/mini");
+
+ ObserverConfiguration doc = new ObserverConfiguration(DocumentObserver.class.getName());
+ props.addObserver(doc);
+
+ SimpleSerializer.setSetserlializer(props, GsonSerializer.class);
+
+ ExportQueue.Options exportQueueOpts =
+ new ExportQueue.Options(RefExporter.QUEUE_ID, RefExporter.class, String.class,
+ RefUpdates.class, getNumBuckets());
+
+ if (getBufferSize() != null) {
+ exportQueueOpts.setBufferSize(getBufferSize());
+ }
+
+ ExportQueue.configure(props, exportQueueOpts);
+
+ miniFluo = FluoFactory.newMiniFluo(props);
+
+ globalExports.clear();
+ exportCalls = 0;
+ }
+
+ @After
+ public void tearDownFluo() throws Exception {
+ if (miniFluo != null) {
+ miniFluo.close();
+ }
+ }
+
+ protected static Set<String> ns(String... sa) {
+ return new HashSet<>(Arrays.asList(sa));
+ }
+
+ protected static String nk(int i) {
+ return String.format("%06d", i);
+ }
+
+ protected static Set<String> ns(int... ia) {
+ HashSet<String> ret = new HashSet<>();
+ for (int i : ia) {
+ ret.add(nk(i));
+ }
+ return ret;
+ }
+
+ public void assertEquals(Map<String, Set<String>> expected, Map<String, Set<String>> actual,
+ FluoClient fc) {
+ if (!expected.equals(actual)) {
+ System.out.println("*** diff ***");
+ diff(expected, actual);
+ System.out.println("*** fluo dump ***");
+ dump(fc);
+ System.out.println("*** map dump ***");
+
+ Assert.fail();
+ }
+ }
+
+ protected void loadRandom(FluoClient fc, int num, int maxDocId) {
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ Random rand = new Random();
+
+ for (int i = 0; i < num; i++) {
+ String docid = String.format("%05d", rand.nextInt(maxDocId));
+ String[] refs = new String[rand.nextInt(20) + 1];
+ for (int j = 0; j < refs.length; j++) {
+ refs[j] = String.format("%05d", rand.nextInt(maxDocId));
+ }
+
+ loader.execute(new DocumentLoader(docid, refs));
+ }
+ }
+ }
+
+ protected void diff(Map<String, Set<String>> fr, Map<String, Set<String>> er) {
+ HashSet<String> allKeys = new HashSet<>(fr.keySet());
+ allKeys.addAll(er.keySet());
+
+ for (String k : allKeys) {
+ Set<String> s1 = fr.getOrDefault(k, Collections.emptySet());
+ Set<String> s2 = er.getOrDefault(k, Collections.emptySet());
+
+ HashSet<String> sub1 = new HashSet<>(s1);
+ sub1.removeAll(s2);
+
+ HashSet<String> sub2 = new HashSet<>(s2);
+ sub2.removeAll(s1);
+
+ if (sub1.size() > 0 || sub2.size() > 0) {
+ System.out.println(k + " " + sub1 + " " + sub2);
+ }
+
+ }
+ }
+
+ protected Map<String, Set<String>> getFluoReferees(FluoClient fc) {
+ Map<String, Set<String>> fluoReferees = new HashMap<>();
+
+ try (Snapshot snap = fc.newSnapshot()) {
+ ScannerConfiguration scannerConfig = new ScannerConfiguration();
+ scannerConfig.fetchColumn(Bytes.of("content"), Bytes.of("current"));
+ scannerConfig.setSpan(Span.prefix("d:"));
+ RowIterator scanner = snap.get(scannerConfig);
+ while (scanner.hasNext()) {
+ Entry<Bytes, ColumnIterator> row = scanner.next();
+ ColumnIterator colIter = row.getValue();
+
+ String docid = row.getKey().toString().substring(2);
+
+ while (colIter.hasNext()) {
+ Entry<Column, Bytes> entry = colIter.next();
+
+ String[] refs = entry.getValue().toString().split(" ");
+
+ for (String ref : refs) {
+ if (ref.isEmpty())
+ continue;
+
+ fluoReferees.computeIfAbsent(ref, k -> new HashSet<>()).add(docid);
+ }
+ }
+ }
+ }
+ return fluoReferees;
+ }
+
+ public static void dump(FluoClient fc) {
+ try (Snapshot snap = fc.newSnapshot()) {
+ RowIterator scanner = snap.get(new ScannerConfiguration());
+ while (scanner.hasNext()) {
+ Entry<Bytes, ColumnIterator> row = scanner.next();
+ ColumnIterator colIter = row.getValue();
+
+ while (colIter.hasNext()) {
+ Entry<Column, Bytes> entry = colIter.next();
+
+ System.out.println("row:[" + row.getKey() + "] col:[" + entry.getKey() + "] val:["
+ + entry.getValue() + "]");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/GsonSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/GsonSerializer.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/GsonSerializer.java
new file mode 100644
index 0000000..2d45ff3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/GsonSerializer.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.nio.charset.StandardCharsets;
+
+import com.google.gson.Gson;
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+public class GsonSerializer implements SimpleSerializer {
+
+ private Gson gson = new Gson();
+
+ @Override
+ public void init(SimpleConfiguration appConfig) {
+
+ }
+
+ @Override
+ public <T> byte[] serialize(T obj) {
+ return gson.toJson(obj).getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public <T> T deserialize(byte[] serObj, Class<T> clazz) {
+ return gson.fromJson(new String(serObj, StandardCharsets.UTF_8), clazz);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/OptionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/OptionsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/OptionsTest.java
new file mode 100644
index 0000000..b07caea
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/OptionsTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.recipes.core.export.ExportQueue.Options;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class OptionsTest {
+ @Test
+ public void testExportQueueOptions() {
+ FluoConfiguration conf = new FluoConfiguration();
+
+ ExportQueue.configure(conf, new Options("Q1", "ET", "KT", "VT", 100));
+ ExportQueue.configure(conf, new Options("Q2", "ET2", "KT2", "VT2", 200).setBucketsPerTablet(20)
+ .setBufferSize(1000000));
+
+ Options opts1 = new Options("Q1", conf.getAppConfiguration());
+
+ Assert.assertEquals(opts1.exporterType, "ET");
+ Assert.assertEquals(opts1.keyType, "KT");
+ Assert.assertEquals(opts1.valueType, "VT");
+ Assert.assertEquals(opts1.numBuckets, 100);
+ Assert.assertEquals(opts1.bucketsPerTablet.intValue(), Options.DEFAULT_BUCKETS_PER_TABLET);
+ Assert.assertEquals(opts1.bufferSize.intValue(), Options.DEFAULT_BUFFER_SIZE);
+
+ Options opts2 = new Options("Q2", conf.getAppConfiguration());
+
+ Assert.assertEquals(opts2.exporterType, "ET2");
+ Assert.assertEquals(opts2.keyType, "KT2");
+ Assert.assertEquals(opts2.valueType, "VT2");
+ Assert.assertEquals(opts2.numBuckets, 200);
+ Assert.assertEquals(opts2.bucketsPerTablet.intValue(), 20);
+ Assert.assertEquals(opts2.bufferSize.intValue(), 1000000);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefInfo.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefInfo.java
new file mode 100644
index 0000000..f4d1c76
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefInfo.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+class RefInfo {
+ long seq;
+ boolean deleted;
+
+ public RefInfo(long seq, boolean deleted) {
+ this.seq = seq;
+ this.deleted = deleted;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefUpdates.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefUpdates.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefUpdates.java
new file mode 100644
index 0000000..efa94dd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/export/RefUpdates.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Set;
+
+public class RefUpdates {
+ private Set<String> addedRefs;
+ private Set<String> deletedRefs;
+
+ public RefUpdates() {}
+
+ public RefUpdates(Set<String> addedRefs, Set<String> deletedRefs) {
+ this.addedRefs = addedRefs;
+ this.deletedRefs = deletedRefs;
+ }
+
+ public Set<String> getAddedRefs() {
+ return addedRefs;
+ }
+
+ public Set<String> getDeletedRefs() {
+ return deletedRefs;
+ }
+
+ @Override
+ public String toString() {
+ return "added:" + addedRefs + " deleted:" + deletedRefs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/BigUpdateIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/BigUpdateIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/BigUpdateIT.java
new file mode 100644
index 0000000..e5f7d55
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/BigUpdateIT.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.api.mini.MiniFluo;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+import org.apache.fluo.recipes.core.types.StringEncoder;
+import org.apache.fluo.recipes.core.types.TypeLayer;
+import org.apache.fluo.recipes.core.types.TypedSnapshot;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This test configures a small buffer size and verifies that multiple passes are made to process
+ * updates.
+ */
+public class BigUpdateIT {
+ private static final TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ private MiniFluo miniFluo;
+
+ private CollisionFreeMap<String, Long> wcMap;
+
+ static final String MAP_ID = "bu";
+
+ public static class LongCombiner implements Combiner<String, Long> {
+
+ @Override
+ public Optional<Long> combine(String key, Iterator<Long> updates) {
+ long[] count = new long[] {0};
+ updates.forEachRemaining(l -> count[0] += l);
+ return Optional.of(count[0]);
+ }
+ }
+
+ static final Column DSCOL = new Column("debug", "sum");
+
+ private static AtomicInteger globalUpdates = new AtomicInteger(0);
+
+ public static class MyObserver extends UpdateObserver<String, Long> {
+
+ @Override
+ public void updatingValues(TransactionBase tx, Iterator<Update<String, Long>> updates) {
+ TypedTransactionBase ttx = tl.wrap(tx);
+
+ Map<String, Long> expectedOld = new HashMap<>();
+
+
+ while (updates.hasNext()) {
+ Update<String, Long> update = updates.next();
+
+ if (update.getOldValue().isPresent()) {
+ expectedOld.put("side:" + update.getKey(), update.getOldValue().get());
+ }
+
+ ttx.mutate().row("side:" + update.getKey()).col(DSCOL).set(update.getNewValue().get());
+ }
+
+ // get last values set to verify same as passed in old value
+ Map<String, Long> actualOld =
+ Maps.transformValues(
+ ttx.get().rowsString(expectedOld.keySet()).columns(ImmutableSet.of(DSCOL))
+ .toStringMap(), m -> m.get(DSCOL).toLong());
+
+ MapDifference<String, Long> diff = Maps.difference(expectedOld, actualOld);
+
+ Assert.assertTrue(diff.toString(), diff.areEqual());
+
+ globalUpdates.incrementAndGet();
+ }
+ }
+
+ @Before
+ public void setUpFluo() throws Exception {
+ FileUtils.deleteQuietly(new File("target/mini"));
+
+ FluoConfiguration props = new FluoConfiguration();
+ props.setApplicationName("eqt");
+ props.setWorkerThreads(20);
+ props.setMiniDataDir("target/mini");
+
+ SimpleSerializer.setSetserlializer(props, TestSerializer.class);
+
+ CollisionFreeMap.configure(props, new CollisionFreeMap.Options(MAP_ID, LongCombiner.class,
+ MyObserver.class, String.class, Long.class, 2).setBufferSize(1 << 10));
+
+ miniFluo = FluoFactory.newMiniFluo(props);
+
+ wcMap = CollisionFreeMap.getInstance(MAP_ID, props.getAppConfiguration());
+
+ globalUpdates.set(0);
+ }
+
+ @After
+ public void tearDownFluo() throws Exception {
+ if (miniFluo != null) {
+ miniFluo.close();
+ }
+ }
+
+ @Test
+ public void testBigUpdates() {
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ updateMany(fc);
+
+ miniFluo.waitForObservers();
+
+ int numUpdates = 0;
+
+ try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
+ checkUpdates(snap, 1, 1000);
+ numUpdates = globalUpdates.get();
+ // there are two buckets, expect update processing at least twice per bucket
+ Assert.assertTrue(numUpdates >= 4);
+ }
+
+ updateMany(fc);
+ updateMany(fc);
+
+ miniFluo.waitForObservers();
+
+ try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
+ checkUpdates(snap, 3, 1000);
+ numUpdates = globalUpdates.get() - numUpdates;
+ Assert.assertTrue(numUpdates >= 4);
+ }
+
+ for (int i = 0; i < 10; i++) {
+ updateMany(fc);
+ }
+
+ miniFluo.waitForObservers();
+
+ try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
+ checkUpdates(snap, 13, 1000);
+ numUpdates = globalUpdates.get() - numUpdates;
+ Assert.assertTrue(numUpdates >= 4);
+ }
+ }
+ }
+
+ private void checkUpdates(TypedSnapshot snap, long expectedVal, long expectedRows) {
+ RowIterator iter = snap.get(new ScannerConfiguration().setSpan(Span.prefix("side:")));
+
+ int row = 0;
+
+ while (iter.hasNext()) {
+ Entry<Bytes, ColumnIterator> entry = iter.next();
+
+ Assert.assertEquals(String.format("side:%06d", row++), entry.getKey().toString());
+
+ ColumnIterator colITer = entry.getValue();
+ while (colITer.hasNext()) {
+ Entry<Column, Bytes> entry2 = colITer.next();
+ Assert.assertEquals(new Column("debug", "sum"), entry2.getKey());
+ Assert.assertEquals("row : " + entry.getKey(), "" + expectedVal, entry2.getValue()
+ .toString());
+ }
+ }
+
+ Assert.assertEquals(expectedRows, row);
+ }
+
+ private void updateMany(FluoClient fc) {
+ try (Transaction tx = fc.newTransaction()) {
+ Map<String, Long> updates = new HashMap<>();
+ for (int i = 0; i < 1000; i++) {
+ updates.put(String.format("%06d", i), 1L);
+ }
+
+ wcMap.update(tx, updates);
+ tx.commit();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/CollisionFreeMapIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/CollisionFreeMapIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/CollisionFreeMapIT.java
new file mode 100644
index 0000000..f7dbc89
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/CollisionFreeMapIT.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.client.LoaderExecutor;
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.ObserverConfiguration;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.api.mini.MiniFluo;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CollisionFreeMapIT {
+
+ private MiniFluo miniFluo;
+
+ private CollisionFreeMap<String, Long> wcMap;
+
+ static final String MAP_ID = "wcm";
+
+ @Before
+ public void setUpFluo() throws Exception {
+ FileUtils.deleteQuietly(new File("target/mini"));
+
+ FluoConfiguration props = new FluoConfiguration();
+ props.setApplicationName("eqt");
+ props.setWorkerThreads(20);
+ props.setMiniDataDir("target/mini");
+
+ props.addObserver(new ObserverConfiguration(DocumentObserver.class.getName()));
+
+ SimpleSerializer.setSetserlializer(props, TestSerializer.class);
+
+ CollisionFreeMap.configure(props, new CollisionFreeMap.Options(MAP_ID, WordCountCombiner.class,
+ WordCountObserver.class, String.class, Long.class, 17));
+
+ miniFluo = FluoFactory.newMiniFluo(props);
+
+ wcMap = CollisionFreeMap.getInstance(MAP_ID, props.getAppConfiguration());
+ }
+
+ @After
+ public void tearDownFluo() throws Exception {
+ if (miniFluo != null) {
+ miniFluo.close();
+ }
+ }
+
+ private Map<String, Long> getComputedWordCounts(FluoClient fc) {
+ Map<String, Long> counts = new HashMap<>();
+
+ try (Snapshot snap = fc.newSnapshot()) {
+ RowIterator scanner = snap.get(new ScannerConfiguration().setSpan(Span.prefix("iwc:")));
+ while (scanner.hasNext()) {
+ Entry<Bytes, ColumnIterator> row = scanner.next();
+
+ String[] tokens = row.getKey().toString().split(":");
+ String word = tokens[2];
+ Long count = Long.valueOf(tokens[1]);
+
+ Assert.assertFalse("Word seen twice in index " + word, counts.containsKey(word));
+
+ counts.put(word, count);
+ }
+ }
+
+ return counts;
+ }
+
+ private Map<String, Long> computeWordCounts(FluoClient fc) {
+ Map<String, Long> counts = new HashMap<>();
+
+ try (Snapshot snap = fc.newSnapshot()) {
+ RowIterator scanner =
+ snap.get(new ScannerConfiguration().setSpan(Span.prefix("d:")).fetchColumn(
+ Bytes.of("content"), Bytes.of("current")));
+ while (scanner.hasNext()) {
+ Entry<Bytes, ColumnIterator> row = scanner.next();
+
+ ColumnIterator colIter = row.getValue();
+
+ while (colIter.hasNext()) {
+ Entry<Column, Bytes> entry = colIter.next();
+
+ String[] words = entry.getValue().toString().split("\\s+");
+ for (String word : words) {
+ if (word.isEmpty()) {
+ continue;
+ }
+
+ counts.merge(word, 1L, Long::sum);
+ }
+ }
+ }
+ }
+
+ return counts;
+ }
+
+ @Test
+ public void testGet() {
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ try (Transaction tx = fc.newTransaction()) {
+ wcMap.update(tx, ImmutableMap.of("cat", 2L, "dog", 5L));
+ tx.commit();
+ }
+
+ try (Transaction tx = fc.newTransaction()) {
+ wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", 1L));
+ tx.commit();
+ }
+
+ try (Transaction tx = fc.newTransaction()) {
+ wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", 1L, "fish", 2L));
+ tx.commit();
+ }
+
+ // try reading possibly before observer combines... will either see outstanding updates or a
+ // current value
+ try (Snapshot snap = fc.newSnapshot()) {
+ Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
+ Assert.assertEquals((Long) 7L, wcMap.get(snap, "dog"));
+ Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
+ }
+
+ miniFluo.waitForObservers();
+
+ // in this case there should be no updates, only a current value
+ try (Snapshot snap = fc.newSnapshot()) {
+ Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
+ Assert.assertEquals((Long) 7L, wcMap.get(snap, "dog"));
+ Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
+ }
+
+ Map<String, Long> expectedCounts = new HashMap<>();
+ expectedCounts.put("cat", 4L);
+ expectedCounts.put("dog", 7L);
+ expectedCounts.put("fish", 2L);
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (Transaction tx = fc.newTransaction()) {
+ wcMap.update(tx, ImmutableMap.of("cat", 1L, "dog", -7L));
+ tx.commit();
+ }
+
+ // there may be outstanding update and a current value for the key in this case
+ try (Snapshot snap = fc.newSnapshot()) {
+ Assert.assertEquals((Long) 5L, wcMap.get(snap, "cat"));
+ Assert.assertNull(wcMap.get(snap, "dog"));
+ Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
+ }
+
+ miniFluo.waitForObservers();
+
+ try (Snapshot snap = fc.newSnapshot()) {
+ Assert.assertEquals((Long) 5L, wcMap.get(snap, "cat"));
+ Assert.assertNull(wcMap.get(snap, "dog"));
+ Assert.assertEquals((Long) 2L, wcMap.get(snap, "fish"));
+ }
+
+ expectedCounts.put("cat", 5L);
+ expectedCounts.remove("dog");
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+ }
+ }
+
+ @Test
+ public void testBasic() {
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0001", "dog cat"));
+ loader.execute(new DocumentLoader("0002", "cat hamster"));
+ loader.execute(new DocumentLoader("0003", "milk bread cat food"));
+ loader.execute(new DocumentLoader("0004", "zoo big cat"));
+ }
+
+ miniFluo.waitForObservers();
+
+ try (Snapshot snap = fc.newSnapshot()) {
+ Assert.assertEquals((Long) 4L, wcMap.get(snap, "cat"));
+ Assert.assertEquals((Long) 1L, wcMap.get(snap, "milk"));
+ }
+
+ Map<String, Long> expectedCounts = new HashMap<>();
+ expectedCounts.put("dog", 1L);
+ expectedCounts.put("cat", 4L);
+ expectedCounts.put("hamster", 1L);
+ expectedCounts.put("milk", 1L);
+ expectedCounts.put("bread", 1L);
+ expectedCounts.put("food", 1L);
+ expectedCounts.put("zoo", 1L);
+ expectedCounts.put("big", 1L);
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0001", "dog feline"));
+ }
+
+ miniFluo.waitForObservers();
+
+ expectedCounts.put("cat", 3L);
+ expectedCounts.put("feline", 1L);
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ // swap contents of two documents... should not change doc counts
+ loader.execute(new DocumentLoader("0003", "zoo big cat"));
+ loader.execute(new DocumentLoader("0004", "milk bread cat food"));
+ }
+
+ miniFluo.waitForObservers();
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0003", "zoo big cat"));
+ loader.execute(new DocumentLoader("0004", "zoo big cat"));
+ }
+
+ miniFluo.waitForObservers();
+
+ expectedCounts.put("zoo", 2L);
+ expectedCounts.put("big", 2L);
+ expectedCounts.remove("milk");
+ expectedCounts.remove("bread");
+ expectedCounts.remove("food");
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0002", "cat cat hamster hamster"));
+ }
+
+ miniFluo.waitForObservers();
+
+ expectedCounts.put("cat", 4L);
+ expectedCounts.put("hamster", 2L);
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ loader.execute(new DocumentLoader("0002", "dog hamster"));
+ }
+
+ miniFluo.waitForObservers();
+
+ expectedCounts.put("cat", 2L);
+ expectedCounts.put("hamster", 1L);
+ expectedCounts.put("dog", 2L);
+
+ Assert.assertEquals(expectedCounts, getComputedWordCounts(fc));
+ }
+ }
+
+ private static String randDocId(Random rand) {
+ return String.format("%04d", rand.nextInt(5000));
+ }
+
+ private static String randomDocument(Random rand) {
+ StringBuilder sb = new StringBuilder();
+
+ String sep = "";
+ for (int i = 2; i < rand.nextInt(18); i++) {
+ sb.append(sep);
+ sep = " ";
+ sb.append(String.format("%05d", rand.nextInt(50000)));
+ }
+
+ return sb.toString();
+ }
+
+ public void diff(Map<String, Long> m1, Map<String, Long> m2) {
+ for (String word : m1.keySet()) {
+ Long v1 = m1.get(word);
+ Long v2 = m2.get(word);
+
+ if (v2 == null || !v1.equals(v2)) {
+ System.out.println(word + " " + v1 + " != " + v2);
+ }
+ }
+
+ for (String word : m2.keySet()) {
+ Long v1 = m1.get(word);
+ Long v2 = m2.get(word);
+
+ if (v1 == null) {
+ System.out.println(word + " null != " + v2);
+ }
+ }
+ }
+
+ @Test
+ public void testStress() throws Exception {
+ try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
+ Random rand = new Random();
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ for (int i = 0; i < 1000; i++) {
+ loader.execute(new DocumentLoader(randDocId(rand), randomDocument(rand)));
+ }
+ }
+
+ miniFluo.waitForObservers();
+ assertWordCountsEqual(fc);
+
+ try (LoaderExecutor loader = fc.newLoaderExecutor()) {
+ for (int i = 0; i < 100; i++) {
+ loader.execute(new DocumentLoader(randDocId(rand), randomDocument(rand)));
+ }
+ }
+
+ miniFluo.waitForObservers();
+ assertWordCountsEqual(fc);
+ }
+ }
+
+ private void assertWordCountsEqual(FluoClient fc) {
+ Map<String, Long> expected = computeWordCounts(fc);
+ Map<String, Long> actual = getComputedWordCounts(fc);
+ if (!expected.equals(actual)) {
+ diff(expected, actual);
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentLoader.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentLoader.java
new file mode 100644
index 0000000..54f5ee1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentLoader.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import org.apache.fluo.recipes.core.types.TypedLoader;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+
+public class DocumentLoader extends TypedLoader {
+
+ String docid;
+ String doc;
+
+ DocumentLoader(String docid, String doc) {
+ this.docid = docid;
+ this.doc = doc;
+ }
+
+ @Override
+ public void load(TypedTransactionBase tx, Context context) throws Exception {
+ tx.mutate().row("d:" + docid).fam("content").qual("new").set(doc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentObserver.java
new file mode 100644
index 0000000..2c79f45
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/DocumentObserver.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.recipes.core.types.TypedObserver;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+
+public class DocumentObserver extends TypedObserver {
+
+ CollisionFreeMap<String, Long> wcm;
+
+ @Override
+ public void init(Context context) throws Exception {
+ wcm = CollisionFreeMap.getInstance(CollisionFreeMapIT.MAP_ID, context.getAppConfiguration());
+ }
+
+ @Override
+ public ObservedColumn getObservedColumn() {
+ return new ObservedColumn(new Column("content", "new"), NotificationType.STRONG);
+ }
+
+ static Map<String, Long> getWordCounts(String doc) {
+ Map<String, Long> wordCounts = new HashMap<>();
+ String[] words = doc.split(" ");
+ for (String word : words) {
+ if (word.isEmpty()) {
+ continue;
+ }
+ wordCounts.merge(word, 1L, Long::sum);
+ }
+
+ return wordCounts;
+ }
+
+ @Override
+ public void process(TypedTransactionBase tx, Bytes row, Column col) {
+ String newContent = tx.get().row(row).col(col).toString();
+ String currentContent = tx.get().row(row).fam("content").qual("current").toString("");
+
+ Map<String, Long> newWordCounts = getWordCounts(newContent);
+ Map<String, Long> currentWordCounts = getWordCounts(currentContent);
+
+ Map<String, Long> changes = calculateChanges(newWordCounts, currentWordCounts);
+
+ wcm.update(tx, changes);
+
+ tx.mutate().row(row).fam("content").qual("current").set(newContent);
+ }
+
+ private static Map<String, Long> calculateChanges(Map<String, Long> newCounts,
+ Map<String, Long> currCounts) {
+ Map<String, Long> changes = new HashMap<>();
+
+ // guava Maps class
+ MapDifference<String, Long> diffs = Maps.difference(currCounts, newCounts);
+
+ // compute the diffs for words that changed
+ changes.putAll(Maps.transformValues(diffs.entriesDiffering(), vDiff -> vDiff.rightValue()
+ - vDiff.leftValue()));
+
+ // add all new words
+ changes.putAll(diffs.entriesOnlyOnRight());
+
+ // subtract all words no longer present
+ changes.putAll(Maps.transformValues(diffs.entriesOnlyOnLeft(), l -> l * -1));
+
+ return changes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/OptionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/OptionsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/OptionsTest.java
new file mode 100644
index 0000000..6587f6e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/OptionsTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.recipes.core.map.CollisionFreeMap.Options;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class OptionsTest {
+ @Test
+ public void testExportQueueOptions() {
+ FluoConfiguration conf = new FluoConfiguration();
+
+ CollisionFreeMap.configure(conf, new Options("Q1", "CT", "KT", "VT", 100));
+ CollisionFreeMap.configure(conf, new Options("Q2", "CT2", "KT2", "VT2", 200)
+ .setBucketsPerTablet(20).setBufferSize(1000000));
+
+ Options opts1 = new Options("Q1", conf.getAppConfiguration());
+
+ Assert.assertEquals(opts1.combinerType, "CT");
+ Assert.assertEquals(opts1.keyType, "KT");
+ Assert.assertEquals(opts1.valueType, "VT");
+ Assert.assertEquals(opts1.numBuckets, 100);
+ Assert.assertEquals(opts1.bucketsPerTablet.intValue(), Options.DEFAULT_BUCKETS_PER_TABLET);
+ Assert.assertEquals(opts1.bufferSize.intValue(), Options.DEFAULT_BUFFER_SIZE);
+
+ Options opts2 = new Options("Q2", conf.getAppConfiguration());
+
+ Assert.assertEquals(opts2.combinerType, "CT2");
+ Assert.assertEquals(opts2.keyType, "KT2");
+ Assert.assertEquals(opts2.valueType, "VT2");
+ Assert.assertEquals(opts2.numBuckets, 200);
+ Assert.assertEquals(opts2.bucketsPerTablet.intValue(), 20);
+ Assert.assertEquals(opts2.bufferSize.intValue(), 1000000);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
new file mode 100644
index 0000000..a359598
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.recipes.core.common.Pirtos;
+import org.apache.fluo.recipes.core.map.CollisionFreeMap.Options;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SplitsTest {
+ private static List<Bytes> sort(List<Bytes> in) {
+ ArrayList<Bytes> out = new ArrayList<>(in);
+ Collections.sort(out);
+ return out;
+ }
+
+ @Test
+ public void testSplits() {
+
+ Options opts = new Options("foo", WordCountCombiner.class, String.class, Long.class, 3);
+ opts.setBucketsPerTablet(1);
+ FluoConfiguration fluoConfig = new FluoConfiguration();
+ CollisionFreeMap.configure(fluoConfig, opts);
+
+ Pirtos pirtos1 =
+ CollisionFreeMap.getTableOptimizations("foo", fluoConfig.getAppConfiguration());
+ List<Bytes> expected1 =
+ Lists.transform(
+ Arrays.asList("foo:d:1", "foo:d:2", "foo:d:~", "foo:u:1", "foo:u:2", "foo:u:~"),
+ Bytes::of);
+
+ Assert.assertEquals(expected1, sort(pirtos1.getSplits()));
+
+ Options opts2 = new Options("bar", WordCountCombiner.class, String.class, Long.class, 6);
+ opts2.setBucketsPerTablet(2);
+ CollisionFreeMap.configure(fluoConfig, opts2);
+
+ Pirtos pirtos2 =
+ CollisionFreeMap.getTableOptimizations("bar", fluoConfig.getAppConfiguration());
+ List<Bytes> expected2 =
+ Lists.transform(
+ Arrays.asList("bar:d:2", "bar:d:4", "bar:d:~", "bar:u:2", "bar:u:4", "bar:u:~"),
+ Bytes::of);
+ Assert.assertEquals(expected2, sort(pirtos2.getSplits()));
+
+ Pirtos pirtos3 = CollisionFreeMap.getTableOptimizations(fluoConfig.getAppConfiguration());
+
+ ArrayList<Bytes> expected3 = new ArrayList<>(expected2);
+ expected3.addAll(expected1);
+
+ Assert.assertEquals(expected3, sort(pirtos3.getSplits()));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/TestSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/TestSerializer.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/TestSerializer.java
new file mode 100644
index 0000000..b59705a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/TestSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+public class TestSerializer implements SimpleSerializer {
+
+ @Override
+ public <T> byte[] serialize(T obj) {
+ return obj.toString().getBytes();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T deserialize(byte[] serObj, Class<T> clazz) {
+ if (clazz.equals(Long.class)) {
+ return (T) Long.valueOf(new String(serObj));
+ }
+
+ if (clazz.equals(String.class)) {
+ return (T) new String(serObj);
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ @Override
+ public void init(SimpleConfiguration appConfig) {}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountCombiner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountCombiner.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountCombiner.java
new file mode 100644
index 0000000..f757c10
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountCombiner.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Iterator;
+import java.util.Optional;
+
+public class WordCountCombiner implements Combiner<String, Long> {
+ @Override
+ public Optional<Long> combine(String key, Iterator<Long> updates) {
+ long sum = 0;
+
+ while (updates.hasNext()) {
+ sum += updates.next();
+ }
+
+ if (sum == 0) {
+ return Optional.empty();
+ } else {
+ return Optional.of(sum);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountObserver.java
new file mode 100644
index 0000000..221083c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/WordCountObserver.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Iterator;
+import java.util.Optional;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+
+public class WordCountObserver extends UpdateObserver<String, Long> {
+
+ @Override
+ public void updatingValues(TransactionBase tx, Iterator<Update<String, Long>> updates) {
+
+ while (updates.hasNext()) {
+ Update<String, Long> update = updates.next();
+
+ Optional<Long> oldVal = update.getOldValue();
+ Optional<Long> newVal = update.getNewValue();
+
+ if (oldVal.isPresent()) {
+ String oldRow = String.format("iwc:%09d:%s", oldVal.get(), update.getKey());
+ tx.delete(Bytes.of(oldRow), new Column(Bytes.EMPTY, Bytes.EMPTY));
+ }
+
+ if (newVal.isPresent()) {
+ String newRow = String.format("iwc:%09d:%s", newVal.get(), update.getKey());
+ tx.set(Bytes.of(newRow), new Column(Bytes.EMPTY, Bytes.EMPTY), Bytes.EMPTY);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionTest.java
new file mode 100644
index 0000000..7c09b6e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.transaction;
+
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.recipes.core.types.StringEncoder;
+import org.apache.fluo.recipes.core.types.TypeLayer;
+import org.apache.fluo.recipes.core.types.TypedTransaction;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.mock;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class RecordingTransactionTest {
+
+ private Transaction tx;
+ private RecordingTransaction rtx;
+ private TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ @Before
+ public void setUp() {
+ tx = mock(Transaction.class);
+ rtx = RecordingTransaction.wrap(tx);
+ }
+
+ @Test
+ public void testTx() {
+ rtx.set(Bytes.of("r1"), new Column("cf1"), Bytes.of("v1"));
+ rtx.set(Bytes.of("r2"), new Column("cf2", "cq2"), Bytes.of("v2"));
+ rtx.delete(Bytes.of("r3"), new Column("cf3"));
+ expect(tx.get(Bytes.of("r4"), new Column("cf4"))).andReturn(Bytes.of("v4"));
+ replay(tx);
+ rtx.get(Bytes.of("r4"), new Column("cf4"));
+
+ List<LogEntry> entries = rtx.getTxLog().getLogEntries();
+ Assert.assertEquals(4, entries.size());
+ Assert.assertEquals("LogEntry{op=SET, row=r1, col=cf1 , value=v1}", entries.get(0).toString());
+ Assert.assertEquals("LogEntry{op=SET, row=r2, col=cf2 cq2 , value=v2}", entries.get(1)
+ .toString());
+ Assert
+ .assertEquals("LogEntry{op=DELETE, row=r3, col=cf3 , value=}", entries.get(2).toString());
+ Assert.assertEquals("LogEntry{op=GET, row=r4, col=cf4 , value=v4}", entries.get(3).toString());
+ Assert.assertEquals("{r4 cf4 =v4}", rtx.getTxLog().getOperationMap(LogEntry.Operation.GET)
+ .toString());
+ Assert.assertEquals("{r2 cf2 cq2 =v2, r1 cf1 =v1}",
+ rtx.getTxLog().getOperationMap(LogEntry.Operation.SET).toString());
+ Assert.assertEquals("{r3 cf3 =}", rtx.getTxLog().getOperationMap(LogEntry.Operation.DELETE)
+ .toString());
+ }
+
+ @Test
+ public void testTypedTx() {
+ TypedTransaction ttx = tl.wrap(rtx);
+ ttx.mutate().row("r5").fam("cf5").qual("cq5").set("1");
+ ttx.mutate().row("r6").fam("cf6").qual("cq6").set("1");
+ List<LogEntry> entries = rtx.getTxLog().getLogEntries();
+ Assert.assertEquals(2, entries.size());
+ Assert.assertEquals("LogEntry{op=SET, row=r5, col=cf5 cq5 , value=1}", entries.get(0)
+ .toString());
+ Assert.assertEquals("LogEntry{op=SET, row=r6, col=cf6 cq6 , value=1}", entries.get(1)
+ .toString());
+ }
+
+ @Test
+ public void testFilter() {
+ rtx = RecordingTransaction.wrap(tx, le -> le.getColumn().getFamily().toString().equals("cfa"));
+ TypedTransaction ttx = tl.wrap(rtx);
+ ttx.mutate().row("r1").fam("cfa").qual("cq1").set("1");
+ ttx.mutate().row("r2").fam("cfb").qual("cq2").set("2");
+ ttx.mutate().row("r3").fam("cfa").qual("cq3").set("3");
+ List<LogEntry> entries = rtx.getTxLog().getLogEntries();
+ Assert.assertEquals(2, entries.size());
+ Assert.assertEquals("LogEntry{op=SET, row=r1, col=cfa cq1 , value=1}", entries.get(0)
+ .toString());
+ Assert.assertEquals("LogEntry{op=SET, row=r3, col=cfa cq3 , value=3}", entries.get(1)
+ .toString());
+ }
+
+ @Test
+ public void testClose() {
+ tx.close();
+ replay(tx);
+ rtx.close();
+ verify(tx);
+ }
+
+ @Test
+ public void testCommit() {
+ tx.commit();
+ replay(tx);
+ rtx.commit();
+ verify(tx);
+ }
+
+ @Test
+ public void testDelete() {
+ tx.delete(Bytes.of("r"), Column.EMPTY);
+ replay(tx);
+ rtx.delete(Bytes.of("r"), Column.EMPTY);
+ verify(tx);
+ }
+
+ @Test
+ public void testGet() {
+ expect(tx.get(Bytes.of("r"), Column.EMPTY)).andReturn(Bytes.of("v"));
+ replay(tx);
+ Assert.assertEquals(Bytes.of("v"), rtx.get(Bytes.of("r"), Column.EMPTY));
+ verify(tx);
+ }
+
+ @Test
+ public void testGetColumns() {
+ expect(tx.get(Bytes.of("r"), Collections.emptySet())).andReturn(Collections.emptyMap());
+ replay(tx);
+ Assert.assertEquals(Collections.emptyMap(), rtx.get(Bytes.of("r"), Collections.emptySet()));
+ verify(tx);
+ }
+
+ @Test
+ public void testGetRows() {
+ expect(tx.get(Collections.emptyList(), Collections.emptySet())).andReturn(
+ Collections.emptyMap());
+ replay(tx);
+ Assert.assertEquals(Collections.emptyMap(),
+ rtx.get(Collections.emptyList(), Collections.emptySet()));
+ verify(tx);
+ }
+
+ @Test
+ public void testGetScanNull() {
+ ScannerConfiguration scanConfig = new ScannerConfiguration();
+ expect(tx.get(scanConfig)).andReturn(null);
+ replay(tx);
+ Assert.assertNull(rtx.get(scanConfig));
+ verify(tx);
+ }
+
+ @Test
+ public void testGetScanIter() {
+ ScannerConfiguration scanConfig = new ScannerConfiguration();
+ expect(tx.get(scanConfig)).andReturn(new RowIterator() {
+
+ private boolean hasNextRow = true;
+
+ @Override
+ public boolean hasNext() {
+ return hasNextRow;
+ }
+
+ @Override
+ public Map.Entry<Bytes, ColumnIterator> next() {
+ hasNextRow = false;
+ return new AbstractMap.SimpleEntry<>(Bytes.of("r7"), new ColumnIterator() {
+
+ private boolean hasNextCol = true;
+
+ @Override
+ public boolean hasNext() {
+ return hasNextCol;
+ }
+
+ @Override
+ public Map.Entry<Column, Bytes> next() {
+ hasNextCol = false;
+ return new AbstractMap.SimpleEntry<>(new Column("cf7", "cq7"), Bytes.of("v7"));
+ }
+ });
+ }
+ });
+ replay(tx);
+ RowIterator rowIter = rtx.get(scanConfig);
+ Assert.assertNotNull(rowIter);
+ Assert.assertTrue(rtx.getTxLog().isEmpty());
+ Assert.assertTrue(rowIter.hasNext());
+ Map.Entry<Bytes, ColumnIterator> rowEntry = rowIter.next();
+ Assert.assertFalse(rowIter.hasNext());
+ Assert.assertEquals(Bytes.of("r7"), rowEntry.getKey());
+ ColumnIterator colIter = rowEntry.getValue();
+ Assert.assertTrue(colIter.hasNext());
+ Assert.assertTrue(rtx.getTxLog().isEmpty());
+ Map.Entry<Column, Bytes> colEntry = colIter.next();
+ Assert.assertFalse(rtx.getTxLog().isEmpty());
+ Assert.assertFalse(colIter.hasNext());
+ Assert.assertEquals(new Column("cf7", "cq7"), colEntry.getKey());
+ Assert.assertEquals(Bytes.of("v7"), colEntry.getValue());
+ List<LogEntry> entries = rtx.getTxLog().getLogEntries();
+ Assert.assertEquals(1, entries.size());
+ Assert.assertEquals("LogEntry{op=GET, row=r7, col=cf7 cq7 , value=v7}", entries.get(0)
+ .toString());
+ verify(tx);
+ }
+
+ @Test
+ public void testGetTimestamp() {
+ expect(tx.getStartTimestamp()).andReturn(5L);
+ replay(tx);
+ Assert.assertEquals(5L, rtx.getStartTimestamp());
+ verify(tx);
+ }
+}
[02/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshot.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshot.java
new file mode 100644
index 0000000..64fa7c2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshot.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.client.Snapshot;
+
+public class MockSnapshot extends MockSnapshotBase implements Snapshot {
+
+ MockSnapshot(String... entries) {
+ super(entries);
+ }
+
+ @Override
+ public void close() {
+ // no resources need to be closed
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshotBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshotBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshotBase.java
new file mode 100644
index 0000000..d31b36c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockSnapshotBase.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.fluo.api.client.SnapshotBase;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumn;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.core.impl.TxStringUtil;
+
+public class MockSnapshotBase implements SnapshotBase {
+
+ final Map<Bytes, Map<Column, Bytes>> getData;
+
+ /**
+ * Initializes {@link #getData} using {@link #toRCVM(String...)}
+ */
+ MockSnapshotBase(String... entries) {
+ getData = toRCVM(entries);
+ }
+
+ @Override
+ public Bytes get(Bytes row, Column column) {
+ Map<Column, Bytes> cols = getData.get(row);
+ if (cols != null) {
+ return cols.get(column);
+ }
+
+ return null;
+ }
+
+ @Override
+ public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
+ Map<Column, Bytes> ret = new HashMap<>();
+ Map<Column, Bytes> cols = getData.get(row);
+ if (cols != null) {
+ for (Column column : columns) {
+ Bytes val = cols.get(column);
+ if (val != null) {
+ ret.put(column, val);
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
+
+ Map<Bytes, Map<Column, Bytes>> ret = new HashMap<>();
+
+ for (Bytes row : rows) {
+ Map<Column, Bytes> colMap = get(row, columns);
+ if (colMap != null && colMap.size() > 0) {
+ ret.put(row, colMap);
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public RowIterator get(ScannerConfiguration config) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * toRCVM stands for "To Row Column Value Map". This is a convenience function that takes strings
+ * of the format {@code <row>,<col fam>:<col qual>[:col vis],
+ * <value>} and generates a row, column, value map.
+ */
+ public static Map<Bytes, Map<Column, Bytes>> toRCVM(String... entries) {
+ Map<Bytes, Map<Column, Bytes>> ret = new HashMap<>();
+
+ for (String entry : entries) {
+ String[] rcv = entry.split(",");
+ if (rcv.length != 3 && !(rcv.length == 2 && entry.trim().endsWith(","))) {
+ throw new IllegalArgumentException(
+ "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
+ }
+
+ Bytes row = Bytes.of(rcv[0]);
+ String[] colFields = rcv[1].split(":");
+
+ Column col;
+ if (colFields.length == 3) {
+ col = new Column(colFields[0], colFields[1], colFields[2]);
+ } else if (colFields.length == 2) {
+ col = new Column(colFields[0], colFields[1]);
+ } else {
+ throw new IllegalArgumentException(
+ "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
+ }
+
+ Bytes val;
+ if (rcv.length == 2) {
+ val = Bytes.EMPTY;
+ } else {
+ val = Bytes.of(rcv[2]);
+ }
+
+ Map<Column, Bytes> cols = ret.get(row);
+ if (cols == null) {
+ cols = new HashMap<>();
+ ret.put(row, cols);
+ }
+
+ cols.put(col, val);
+ }
+ return ret;
+ }
+
+ /**
+ * toRCM stands for "To Row Column Map". This is a convenience function that takes strings of the
+ * format {@code <row>,<col fam>:<col qual>[:col vis]} and generates a row, column map.
+ */
+ public static Map<Bytes, Set<Column>> toRCM(String... entries) {
+ Map<Bytes, Set<Column>> ret = new HashMap<>();
+
+ for (String entry : entries) {
+ String[] rcv = entry.split(",");
+ if (rcv.length != 2) {
+ throw new IllegalArgumentException(
+ "expected <row>,<col fam>:<col qual>[:col vis] but saw : " + entry);
+ }
+
+ Bytes row = Bytes.of(rcv[0]);
+ String[] colFields = rcv[1].split(":");
+
+ Column col;
+ if (colFields.length == 3) {
+ col = new Column(colFields[0], colFields[1], colFields[2]);
+ } else if (colFields.length == 2) {
+ col = new Column(colFields[0], colFields[1]);
+ } else {
+ throw new IllegalArgumentException(
+ "expected <row>,<col fam>:<col qual>[:col vis],<value> but saw : " + entry);
+ }
+
+ Set<Column> cols = ret.get(row);
+ if (cols == null) {
+ cols = new HashSet<>();
+ ret.put(row, cols);
+ }
+
+ cols.add(col);
+ }
+ return ret;
+ }
+
+ @Override
+ public long getStartTimestamp() {
+ throw new UnsupportedOperationException();
+ }
+
+
+ @Override
+ public String gets(String row, Column column) {
+ return TxStringUtil.gets(this, row, column);
+ }
+
+ @Override
+ public Map<Column, String> gets(String row, Set<Column> columns) {
+ return TxStringUtil.gets(this, row, columns);
+ }
+
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
+ return TxStringUtil.gets(this, rows, columns);
+ }
+
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
+ return TxStringUtil.gets(this, rowColumns);
+ }
+
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransaction.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransaction.java
new file mode 100644
index 0000000..750e0ee
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransaction.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.exceptions.CommitException;
+
+public class MockTransaction extends MockTransactionBase implements Transaction {
+
+ MockTransaction(String... entries) {
+ super(entries);
+ }
+
+ @Override
+ public void commit() throws CommitException {
+ // does nothing
+ }
+
+ @Override
+ public void close() {
+ // no resources to close
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransactionBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransactionBase.java
new file mode 100644
index 0000000..05ab87e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/MockTransactionBase.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.exceptions.AlreadySetException;
+
+/**
+ * A very simple implementation of {@link TransactionBase} used for testing. All reads are serviced
+ * from {@link #getData}. Updates are stored in {@link #setData}, {@link #deletes}, or
+ * {@link #weakNotifications} depending on the update type.
+ */
+public class MockTransactionBase extends MockSnapshotBase implements TransactionBase {
+
+ final Map<Bytes, Map<Column, Bytes>> setData = new HashMap<>();
+ final Map<Bytes, Set<Column>> deletes = new HashMap<>();
+ final Map<Bytes, Set<Column>> weakNotifications = new HashMap<>();
+
+ MockTransactionBase(String... entries) {
+ super(entries);
+ }
+
+ @Override
+ public void setWeakNotification(Bytes row, Column col) {
+ Set<Column> cols = weakNotifications.get(row);
+ if (cols == null) {
+ cols = new HashSet<>();
+ weakNotifications.put(row, cols);
+ }
+
+ cols.add(col);
+ }
+
+ @Override
+ public void set(Bytes row, Column col, Bytes value) {
+ Map<Column, Bytes> cols = setData.get(row);
+ if (cols == null) {
+ cols = new HashMap<>();
+ setData.put(row, cols);
+ }
+
+ cols.put(col, value);
+ }
+
+ @Override
+ public void delete(Bytes row, Column col) {
+ Set<Column> cols = deletes.get(row);
+ if (cols == null) {
+ cols = new HashSet<>();
+ deletes.put(row, cols);
+ }
+
+ cols.add(col);
+ }
+
+ @Override
+ public void setWeakNotification(String row, Column col) {
+ setWeakNotification(Bytes.of(row), col);
+ }
+
+ @Override
+ public void set(String row, Column col, String value) throws AlreadySetException {
+ set(Bytes.of(row), col, Bytes.of(value));
+ }
+
+ @Override
+ public void delete(String row, Column col) {
+ delete(Bytes.of(row), col);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/core/types/TypeLayerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/core/types/TypeLayerTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/TypeLayerTest.java
new file mode 100644
index 0000000..6b38e12
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/types/TypeLayerTest.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.recipes.core.types.TypedSnapshotBase.Value;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TypeLayerTest {
+
+ @Test
+ public void testColumns() throws Exception {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockTransactionBase tt =
+ new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
+ "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
+
+ TypedTransactionBase ttx = tl.wrap(tt);
+
+ Map<Column, Value> results =
+ ttx.get().row("r2")
+ .columns(ImmutableSet.of(new Column("cf2", "6"), new Column("cf2", "7")));
+
+ Assert.assertNull(results.get(new Column("cf2", "6")).toInteger());
+ Assert.assertEquals(0, results.get(new Column("cf2", "6")).toInteger(0));
+ Assert.assertEquals(12, (int) results.get(new Column("cf2", "7")).toInteger());
+ Assert.assertEquals(12, results.get(new Column("cf2", "7")).toInteger(0));
+
+ Assert.assertEquals(1, results.size());
+
+ results =
+ ttx.get()
+ .row("r2")
+ .columns(
+ ImmutableSet.of(new Column("cf2", "6"), new Column("cf2", "7"), new Column("cf2",
+ "8")));
+
+ Assert.assertNull(results.get(new Column("cf2", "6")).toInteger());
+ Assert.assertEquals(0, results.get(new Column("cf2", "6")).toInteger(0));
+ Assert.assertEquals(12, (int) results.get(new Column("cf2", "7")).toInteger());
+ Assert.assertEquals(12, results.get(new Column("cf2", "7")).toInteger(0));
+ Assert.assertEquals(13, (int) results.get(new Column("cf2", "8")).toInteger());
+ Assert.assertEquals(13, results.get(new Column("cf2", "8")).toInteger(0));
+
+ Assert.assertEquals(2, results.size());
+
+ // test var args
+ Map<Column, Value> results2 =
+ ttx.get().row("r2")
+ .columns(new Column("cf2", "6"), new Column("cf2", "7"), new Column("cf2", "8"));
+ Assert.assertEquals(results, results2);
+ }
+
+ @Test
+ public void testVis() throws Exception {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockTransactionBase tt = new MockTransactionBase("r1,cf1:cq1:A,v1", "r1,cf1:cq2:A&B,v2");
+
+ TypedTransactionBase ttx = tl.wrap(tt);
+
+ Assert.assertNull(ttx.get().row("r1").fam("cf1").qual("cq1").toString());
+ Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A").toString());
+ Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A".getBytes())
+ .toString());
+ Assert.assertEquals("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A"))
+ .toString());
+ Assert.assertEquals("v1",
+ ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A".getBytes())).toString());
+
+ Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B").toString());
+ Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B".getBytes())
+ .toString());
+ Assert.assertNull("v1", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&B"))
+ .toString());
+ Assert.assertNull("v1",
+ ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&B".getBytes()))
+ .toString());
+
+ Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B").toString("v3"));
+ Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis("A&B".getBytes())
+ .toString("v3"));
+ Assert.assertEquals("v3", ttx.get().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&B"))
+ .toString("v3"));
+ Assert.assertEquals(
+ "v3",
+ ttx.get().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&B".getBytes()))
+ .toString("v3"));
+
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&B").set(3);
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&C".getBytes()).set(4);
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&D")).set(5);
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&F".getBytes())).set(7);
+
+ Assert.assertEquals(MockTransactionBase.toRCVM("r1,cf1:cq1:A&B,3", "r1,cf1:cq1:A&C,4",
+ "r1,cf1:cq1:A&D,5", "r1,cf1:cq1:A&F,7"), tt.setData);
+ tt.setData.clear();
+
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&B").delete();
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis("A&C".getBytes()).delete();
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(Bytes.of("A&D")).delete();
+ ttx.mutate().row("r1").fam("cf1").qual("cq1").vis(ByteBuffer.wrap("A&F".getBytes())).delete();
+
+ Assert.assertEquals(MockTransactionBase.toRCM("r1,cf1:cq1:A&B", "r1,cf1:cq1:A&C",
+ "r1,cf1:cq1:A&D", "r1,cf1:cq1:A&F"), tt.deletes);
+ tt.deletes.clear();
+ Assert.assertEquals(0, tt.setData.size());
+ Assert.assertEquals(0, tt.weakNotifications.size());
+
+ }
+
+ @Test
+ public void testBuildColumn() {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ Assert.assertEquals(new Column("f0", "q0"), tl.bc().fam("f0".getBytes()).qual("q0".getBytes())
+ .vis());
+ Assert.assertEquals(new Column("f0", "q0"), tl.bc().fam("f0").qual("q0").vis());
+ Assert.assertEquals(new Column("5", "7"), tl.bc().fam(5).qual(7).vis());
+ Assert.assertEquals(new Column("5", "7"), tl.bc().fam(5l).qual(7l).vis());
+ Assert.assertEquals(new Column("5", "7"), tl.bc().fam(Bytes.of("5")).qual(Bytes.of("7")).vis());
+ Assert.assertEquals(new Column("5", "7"),
+ tl.bc().fam(ByteBuffer.wrap("5".getBytes())).qual(ByteBuffer.wrap("7".getBytes())).vis());
+
+ Assert.assertEquals(new Column("f0", "q0", "A&B"),
+ tl.bc().fam("f0".getBytes()).qual("q0".getBytes()).vis("A&B"));
+ Assert.assertEquals(new Column("f0", "q0", "A&C"),
+ tl.bc().fam("f0").qual("q0").vis("A&C".getBytes()));
+ Assert.assertEquals(new Column("5", "7", "A&D"), tl.bc().fam(5).qual(7).vis(Bytes.of("A&D")));
+ Assert.assertEquals(new Column("5", "7", "A&D"),
+ tl.bc().fam(5).qual(7).vis(ByteBuffer.wrap("A&D".getBytes())));
+ }
+
+ @Test
+ public void testRead() throws Exception {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockSnapshot ms =
+ new MockSnapshot("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
+ "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20",
+ "r3,cf3:cq3,28.195", "r4,cf4:cq4,true");
+
+ TypedSnapshot tts = tl.wrap(ms);
+
+ Assert.assertEquals("v1", tts.get().row("r1").fam("cf1").qual("cq1").toString());
+ Assert.assertEquals("v1", tts.get().row("r1").fam("cf1").qual("cq1").toString("b"));
+ Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual("8").toString());
+ Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual("8").toString("b"));
+ Assert.assertEquals("28.195", tts.get().row("r3").fam("cf3").qual("cq3").toString());
+ Assert.assertEquals("28.195", tts.get().row("r3").fam("cf3").qual("cq3").toString("b"));
+ Assert.assertEquals("true", tts.get().row("r4").fam("cf4").qual("cq4").toString());
+ Assert.assertEquals("true", tts.get().row("r4").fam("cf4").qual("cq4").toString("b"));
+
+ // try converting to different types
+ Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual(8).toString());
+ Assert.assertEquals("13", tts.get().row("r2").fam("cf2").qual(8).toString("b"));
+ Assert.assertEquals((Integer) 13, tts.get().row("r2").fam("cf2").qual(8).toInteger());
+ Assert.assertEquals(13, tts.get().row("r2").fam("cf2").qual(8).toInteger(14));
+ Assert.assertEquals((Long) 13l, tts.get().row("r2").fam("cf2").qual(8).toLong());
+ Assert.assertEquals(13l, tts.get().row("r2").fam("cf2").qual(8).toLong(14l));
+ Assert.assertEquals("13", new String(tts.get().row("r2").fam("cf2").qual(8).toBytes()));
+ Assert.assertEquals("13",
+ new String(tts.get().row("r2").fam("cf2").qual(8).toBytes("14".getBytes())));
+ Assert
+ .assertEquals("13", new String(tts.get().row("r2").col(new Column("cf2", "8")).toBytes()));
+ Assert.assertEquals("13",
+ new String(tts.get().row("r2").col(new Column("cf2", "8")).toBytes("14".getBytes())));
+ Assert.assertEquals("13",
+ Bytes.of(tts.get().row("r2").col(new Column("cf2", "8")).toByteBuffer()).toString());
+ Assert.assertEquals(
+ "13",
+ Bytes.of(
+ tts.get().row("r2").col(new Column("cf2", "8"))
+ .toByteBuffer(ByteBuffer.wrap("14".getBytes()))).toString());
+
+ // test non-existent
+ Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toInteger());
+ Assert.assertEquals(14, tts.get().row("r2").fam("cf3").qual(8).toInteger(14));
+ Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toLong());
+ Assert.assertEquals(14l, tts.get().row("r2").fam("cf3").qual(8).toLong(14l));
+ Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toString());
+ Assert.assertEquals("14", tts.get().row("r2").fam("cf3").qual(8).toString("14"));
+ Assert.assertNull(tts.get().row("r2").fam("cf3").qual(8).toBytes());
+ Assert.assertEquals("14",
+ new String(tts.get().row("r2").fam("cf3").qual(8).toBytes("14".getBytes())));
+ Assert.assertNull(tts.get().row("r2").col(new Column("cf3", "8")).toBytes());
+ Assert.assertEquals("14",
+ new String(tts.get().row("r2").col(new Column("cf3", "8")).toBytes("14".getBytes())));
+ Assert.assertNull(tts.get().row("r2").col(new Column("cf3", "8")).toByteBuffer());
+ Assert.assertEquals(
+ "14",
+ Bytes.of(
+ tts.get().row("r2").col(new Column("cf3", "8"))
+ .toByteBuffer(ByteBuffer.wrap("14".getBytes()))).toString());
+
+ // test float & double
+ Assert.assertEquals((Float) 28.195f, tts.get().row("r3").fam("cf3").qual("cq3").toFloat());
+ Assert.assertEquals(28.195f, tts.get().row("r3").fam("cf3").qual("cq3").toFloat(39.383f), 0.0);
+ Assert.assertEquals((Double) 28.195d, tts.get().row("r3").fam("cf3").qual("cq3").toDouble());
+ Assert.assertEquals(28.195d, tts.get().row("r3").fam("cf3").qual("cq3").toDouble(39.383d), 0.0);
+
+ // test boolean
+ Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean());
+ Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean());
+ Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean(false));
+ Assert.assertEquals(true, tts.get().row("r4").fam("cf4").qual("cq4").toBoolean(false));
+
+ // try different types for row
+ Assert.assertEquals("20", tts.get().row(13).fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row(13l).fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13".getBytes()).fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row(ByteBuffer.wrap("13".getBytes())).fam("9").qual("17")
+ .toString());
+
+ // try different types for cf
+ Assert.assertEquals("20", tts.get().row("13").fam(9).qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam(9l).qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9".getBytes()).qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam(ByteBuffer.wrap("9".getBytes())).qual("17")
+ .toString());
+
+ // try different types for cq
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17").toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual(17l).toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual(17).toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual("17".getBytes()).toString());
+ Assert.assertEquals("20", tts.get().row("13").fam("9").qual(ByteBuffer.wrap("17".getBytes()))
+ .toString());
+
+ ms.close();
+ tts.close();
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockTransactionBase tt =
+ new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
+ "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
+
+ TypedTransactionBase ttx = tl.wrap(tt);
+
+ // test increments data
+ ttx.mutate().row("13").fam("9").qual("17").increment(1);
+ ttx.mutate().row("13").fam("9").qual(18).increment(2);
+ ttx.mutate().row("13").fam("9").qual(19l).increment(3);
+ ttx.mutate().row("13").fam("9").qual("20".getBytes()).increment(4);
+ ttx.mutate().row("13").fam("9").qual(Bytes.of("21")).increment(5); // increment non existent
+ ttx.mutate().row("13").col(new Column("9", "22")).increment(6); // increment non existent
+ ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("23".getBytes())).increment(7); // increment
+ // non
+ // existent
+
+ Assert.assertEquals(MockTransactionBase.toRCVM("13,9:17,21", "13,9:18,22", "13,9:19,23",
+ "13,9:20,24", "13,9:21,5", "13,9:22,6", "13,9:23,7"), tt.setData);
+ tt.setData.clear();
+
+ // test increments long
+ ttx.mutate().row("13").fam("9").qual("17").increment(1l);
+ ttx.mutate().row("13").fam("9").qual(18).increment(2l);
+ ttx.mutate().row("13").fam("9").qual(19l).increment(3l);
+ ttx.mutate().row("13").fam("9").qual("20".getBytes()).increment(4l);
+ ttx.mutate().row("13").fam("9").qual(Bytes.of("21")).increment(5l); // increment non existent
+ ttx.mutate().row("13").col(new Column("9", "22")).increment(6l); // increment non existent
+ ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("23".getBytes())).increment(7l); // increment
+ // non
+ // existent
+
+ Assert.assertEquals(MockTransactionBase.toRCVM("13,9:17,21", "13,9:18,22", "13,9:19,23",
+ "13,9:20,24", "13,9:21,5", "13,9:22,6", "13,9:23,7"), tt.setData);
+ tt.setData.clear();
+
+ // test setting data
+ ttx.mutate().row("13").fam("9").qual("16").set();
+ ttx.mutate().row("13").fam("9").qual("17").set(3);
+ ttx.mutate().row("13").fam("9").qual(18).set(4l);
+ ttx.mutate().row("13").fam("9").qual(19l).set("5");
+ ttx.mutate().row("13").fam("9").qual("20".getBytes()).set("6".getBytes());
+ ttx.mutate().row("13").col(new Column("9", "21")).set("7".getBytes());
+ ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes()))
+ .set(ByteBuffer.wrap("8".getBytes()));
+ ttx.mutate().row("13").fam("9").qual("23").set(2.54f);
+ ttx.mutate().row("13").fam("9").qual("24").set(-6.135d);
+ ttx.mutate().row("13").fam("9").qual("25").set(false);
+
+ Assert.assertEquals(MockTransactionBase.toRCVM("13,9:16,", "13,9:17,3", "13,9:18,4",
+ "13,9:19,5", "13,9:20,6", "13,9:21,7", "13,9:22,8", "13,9:23,2.54", "13,9:24,-6.135",
+ "13,9:25,false"), tt.setData);
+ tt.setData.clear();
+
+ // test deleting data
+ ttx.mutate().row("13").fam("9").qual("17").delete();
+ ttx.mutate().row("13").fam("9").qual(18).delete();
+ ttx.mutate().row("13").fam("9").qual(19l).delete();
+ ttx.mutate().row("13").fam("9").qual("20".getBytes()).delete();
+ ttx.mutate().row("13").col(new Column("9", "21")).delete();
+ ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes())).delete();
+
+ Assert
+ .assertEquals(MockTransactionBase.toRCM("13,9:17", "13,9:18", "13,9:19", "13,9:20",
+ "13,9:21", "13,9:22"), tt.deletes);
+ tt.deletes.clear();
+ Assert.assertEquals(0, tt.setData.size());
+ Assert.assertEquals(0, tt.weakNotifications.size());
+
+ // test weak notifications
+ ttx.mutate().row("13").fam("9").qual("17").weaklyNotify();
+ ttx.mutate().row("13").fam("9").qual(18).weaklyNotify();
+ ttx.mutate().row("13").fam("9").qual(19l).weaklyNotify();
+ ttx.mutate().row("13").fam("9").qual("20".getBytes()).weaklyNotify();
+ ttx.mutate().row("13").col(new Column("9", "21")).weaklyNotify();
+ ttx.mutate().row("13").fam("9").qual(ByteBuffer.wrap("22".getBytes())).weaklyNotify();
+
+ Assert
+ .assertEquals(MockTransactionBase.toRCM("13,9:17", "13,9:18", "13,9:19", "13,9:20",
+ "13,9:21", "13,9:22"), tt.weakNotifications);
+ tt.weakNotifications.clear();
+ Assert.assertEquals(0, tt.setData.size());
+ Assert.assertEquals(0, tt.deletes.size());
+ }
+
+ @Test
+ public void testMultiRow() throws Exception {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockTransactionBase tt =
+ new MockTransactionBase("11,cf1:cq1,1", "11,cf1:cq2,2", "12,cf1:cq1,3", "12,cf1:cq2,4",
+ "13,cf1:cq1,5", "13,cf1:cq2,6");
+
+ TypedTransactionBase ttx = tl.wrap(tt);
+
+ Bytes br1 = Bytes.of("11");
+ Bytes br2 = Bytes.of("12");
+ Bytes br3 = Bytes.of("13");
+
+ Column c1 = new Column("cf1", "cq1");
+ Column c2 = new Column("cf1", "cq2");
+
+ Map<Bytes, Map<Column, Value>> map1 =
+ ttx.get().rows(Arrays.asList(br1, br2)).columns(c1).toBytesMap();
+
+ Assert.assertEquals(map1, ttx.get().rows(br1, br2).columns(c1).toBytesMap());
+
+ Assert.assertEquals("1", map1.get(br1).get(c1).toString());
+ Assert.assertEquals("1", map1.get(br1).get(c1).toString("5"));
+ Assert.assertEquals((Long) (1l), map1.get(br1).get(c1).toLong());
+ Assert.assertEquals(1l, map1.get(br1).get(c1).toLong(5));
+ Assert.assertEquals((Integer) (1), map1.get(br1).get(c1).toInteger());
+ Assert.assertEquals(1, map1.get(br1).get(c1).toInteger(5));
+
+ Assert.assertEquals("5", map1.get(br3).get(c1).toString("5"));
+ Assert.assertNull(map1.get(br3).get(c1).toString());
+ Assert.assertEquals(5l, map1.get(br3).get(c1).toLong(5l));
+ Assert.assertNull(map1.get(br3).get(c1).toLong());
+ Assert.assertEquals(5, map1.get(br1).get(c2).toInteger(5));
+ Assert.assertNull(map1.get(br1).get(c2).toInteger());
+
+ Assert.assertEquals(2, map1.size());
+ Assert.assertEquals(1, map1.get(br1).size());
+ Assert.assertEquals(1, map1.get(br2).size());
+ Assert.assertEquals("3", map1.get(br2).get(c1).toString());
+
+ Map<String, Map<Column, Value>> map2 =
+ ttx.get().rowsString(Arrays.asList("11", "13")).columns(c1).toStringMap();
+
+ Assert.assertEquals(map2, ttx.get().rowsString("11", "13").columns(c1).toStringMap());
+
+ Assert.assertEquals(2, map2.size());
+ Assert.assertEquals(1, map2.get("11").size());
+ Assert.assertEquals(1, map2.get("13").size());
+ Assert.assertEquals((Long) (1l), map2.get("11").get(c1).toLong());
+ Assert.assertEquals(5l, map2.get("13").get(c1).toLong(6));
+
+ Map<Long, Map<Column, Value>> map3 =
+ ttx.get().rowsLong(Arrays.asList(11l, 13l)).columns(c1).toLongMap();
+
+ Assert.assertEquals(map3, ttx.get().rowsLong(11l, 13l).columns(c1).toLongMap());
+
+ Assert.assertEquals(2, map3.size());
+ Assert.assertEquals(1, map3.get(11l).size());
+ Assert.assertEquals(1, map3.get(13l).size());
+ Assert.assertEquals((Long) (1l), map3.get(11l).get(c1).toLong());
+ Assert.assertEquals(5l, map3.get(13l).get(c1).toLong(6));
+
+ Map<Integer, Map<Column, Value>> map4 =
+ ttx.get().rowsInteger(Arrays.asList(11, 13)).columns(c1).toIntegerMap();
+
+ Assert.assertEquals(map4, ttx.get().rowsInteger(11, 13).columns(c1).toIntegerMap());
+
+ Assert.assertEquals(2, map4.size());
+ Assert.assertEquals(1, map4.get(11).size());
+ Assert.assertEquals(1, map4.get(13).size());
+ Assert.assertEquals((Long) (1l), map4.get(11).get(c1).toLong());
+ Assert.assertEquals(5l, map4.get(13).get(c1).toLong(6));
+
+ Map<Integer, Map<Column, Value>> map5 =
+ ttx.get().rowsBytes(Arrays.asList("11".getBytes(), "13".getBytes())).columns(c1)
+ .toIntegerMap();
+
+ Assert.assertEquals(map5, ttx.get().rowsBytes("11".getBytes(), "13".getBytes()).columns(c1)
+ .toIntegerMap());
+
+ Assert.assertEquals(2, map5.size());
+ Assert.assertEquals(1, map5.get(11).size());
+ Assert.assertEquals(1, map5.get(13).size());
+ Assert.assertEquals((Long) (1l), map5.get(11).get(c1).toLong());
+ Assert.assertEquals(5l, map5.get(13).get(c1).toLong(6));
+
+ Map<Integer, Map<Column, Value>> map6 =
+ ttx.get()
+ .rowsByteBuffers(
+ Arrays.asList(ByteBuffer.wrap("11".getBytes()), ByteBuffer.wrap("13".getBytes())))
+ .columns(c1).toIntegerMap();
+
+ Assert.assertEquals(
+ map6,
+ ttx.get()
+ .rowsByteBuffers(ByteBuffer.wrap("11".getBytes()), ByteBuffer.wrap("13".getBytes()))
+ .columns(c1).toIntegerMap());
+
+ Assert.assertEquals(2, map6.size());
+ Assert.assertEquals(1, map6.get(11).size());
+ Assert.assertEquals(1, map6.get(13).size());
+ Assert.assertEquals((Long) (1l), map6.get(11).get(c1).toLong());
+ Assert.assertEquals(5l, map6.get(13).get(c1).toLong(6));
+
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ TypeLayer tl = new TypeLayer(new StringEncoder());
+
+ MockTransactionBase tt =
+ new MockTransactionBase("r1,cf1:cq1,v1", "r1,cf1:cq2,v2", "r1,cf1:cq3,9", "r2,cf2:7,12",
+ "r2,cf2:8,13", "13,9:17,20", "13,9:18,20", "13,9:19,20", "13,9:20,20");
+
+ TypedTransactionBase ttx = tl.wrap(tt);
+
+ Assert.assertEquals(Bytes.of("12"), ttx.get(Bytes.of("r2"), new Column("cf2", "7")));
+ Assert.assertNull(ttx.get(Bytes.of("r2"), new Column("cf2", "9")));
+
+ Map<Column, Bytes> map =
+ ttx.get(Bytes.of("r2"), ImmutableSet.of(new Column("cf2", "7"), new Column("cf2", "8")));
+ Assert.assertEquals(2, map.size());
+ Assert.assertEquals("12", map.get(new Column("cf2", "7")).toString());
+ Assert.assertEquals("13", map.get(new Column("cf2", "8")).toString());
+
+ map = ttx.get(Bytes.of("r6"), ImmutableSet.of(new Column("cf2", "7"), new Column("cf2", "8")));
+ Assert.assertEquals(0, map.size());
+
+ ttx.set(Bytes.of("r6"), new Column("cf2", "7"), Bytes.of("3"));
+ Assert.assertEquals(MockTransactionBase.toRCVM("r6,cf2:7,3"), tt.setData);
+ tt.setData.clear();
+
+ Map<Bytes, Map<Column, Bytes>> map2 =
+ ttx.get(ImmutableSet.of(Bytes.of("r1"), Bytes.of("r2")),
+ ImmutableSet.of(new Column("cf1", "cq1"), new Column("cf2", "8")));
+ Assert.assertEquals(MockTransactionBase.toRCVM("r1,cf1:cq1,v1", "r2,cf2:8,13"), map2);
+
+ ttx.delete(Bytes.of("r6"), new Column("cf2", "7"));
+ Assert.assertEquals(MockTransactionBase.toRCM("r6,cf2:7"), tt.deletes);
+ tt.deletes.clear();
+
+ ttx.setWeakNotification(Bytes.of("r6"), new Column("cf2", "8"));
+ Assert.assertEquals(MockTransactionBase.toRCM("r6,cf2:8"), tt.weakNotifications);
+ tt.weakNotifications.clear();
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/data/RowHasherTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/data/RowHasherTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/data/RowHasherTest.java
deleted file mode 100644
index 3d52c51..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/data/RowHasherTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.data;
-
-import org.apache.fluo.api.data.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class RowHasherTest {
-
- @Test
- public void testBadPrefixes() {
- String[] badPrefixes =
- {"q:she6:test1", "q:she6:test1", "p:Mhe6:test1", "p;she6:test1", "p:she6;test1",
- "p;she6;test1", "p:+he6:test1", "p:s?e6:test1", "p:sh{6:test1", "p:sh6:"};
-
- RowHasher rh = new RowHasher("p");
- for (String badPrefix : badPrefixes) {
- try {
- rh.removeHash(Bytes.of(badPrefix));
- Assert.fail();
- } catch (IllegalArgumentException e) {
- }
- }
- }
-
- @Test
- public void testBasic() {
- RowHasher rh = new RowHasher("p");
- Assert.assertTrue(rh.removeHash(rh.addHash("abc")).toString().equals("abc"));
- rh = new RowHasher("p2");
- Assert.assertTrue(rh.removeHash(rh.addHash("abc")).toString().equals("abc"));
-
- Assert.assertTrue(rh.addHash("abc").toString().startsWith("p2:"));
-
- // test to ensure hash is stable over time
- Assert.assertEquals("p2:she6:test1", rh.addHash("test1").toString());
- Assert.assertEquals("p2:hgt0:0123456789abcdefghijklmnopqrstuvwxyz",
- rh.addHash("0123456789abcdefghijklmnopqrstuvwxyz").toString());
- Assert.assertEquals("p2:fluo:86ce3b094982c6a", rh.addHash("86ce3b094982c6a").toString());
- }
-
- @Test
- public void testBalancerRegex() {
- RowHasher rh = new RowHasher("p");
- String regex = rh.getTableOptimizations(3).getTabletGroupingRegex();
- Assert.assertEquals("(\\Qp:\\E).*", regex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentLoader.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentLoader.java
deleted file mode 100644
index 8fe2b19..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentLoader.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.fluo.recipes.types.TypedLoader;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-
-public class DocumentLoader extends TypedLoader {
-
- String docid;
- String refs[];
-
- DocumentLoader(String docid, String... refs) {
- this.docid = docid;
- this.refs = refs;
- }
-
- @Override
- public void load(TypedTransactionBase tx, Context context) throws Exception {
- tx.mutate().row("d:" + docid).fam("content").qual("new").set(StringUtils.join(refs, " "));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentObserver.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentObserver.java
deleted file mode 100644
index a31461c..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/DocumentObserver.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.recipes.export.ExportTestBase.RefExporter;
-import org.apache.fluo.recipes.types.TypedObserver;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-
-public class DocumentObserver extends TypedObserver {
-
- ExportQueue<String, RefUpdates> refExportQueue;
-
- @Override
- public void init(Context context) throws Exception {
- refExportQueue = ExportQueue.getInstance(RefExporter.QUEUE_ID, context.getAppConfiguration());
- }
-
- @Override
- public ObservedColumn getObservedColumn() {
- return new ObservedColumn(new Column("content", "new"), NotificationType.STRONG);
- }
-
- @Override
- public void process(TypedTransactionBase tx, Bytes row, Column col) {
- String newContent = tx.get().row(row).col(col).toString();
- Set<String> newRefs = new HashSet<>(Arrays.asList(newContent.split(" ")));
- Set<String> currentRefs =
- new HashSet<>(Arrays.asList(tx.get().row(row).fam("content").qual("current").toString("")
- .split(" ")));
-
- Set<String> addedRefs = new HashSet<>(newRefs);
- addedRefs.removeAll(currentRefs);
-
- Set<String> deletedRefs = new HashSet<>(currentRefs);
- deletedRefs.removeAll(newRefs);
-
- String key = row.toString().substring(2);
- RefUpdates val = new RefUpdates(addedRefs, deletedRefs);
-
- refExportQueue.add(tx, key, val);
-
- tx.mutate().row(row).fam("content").qual("current").set(newContent);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportBufferIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportBufferIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportBufferIT.java
deleted file mode 100644
index 53d838d..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportBufferIT.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.client.Transaction;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ExportBufferIT extends ExportTestBase {
-
- @Override
- protected int getNumBuckets() {
- return 2;
- }
-
- @Override
- protected Integer getBufferSize() {
- return 1024;
- }
-
- @Test
- public void testSmallExportBuffer() {
- // try setting the export buffer size small. Make sure everything is exported.
-
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- ExportQueue<String, RefUpdates> refExportQueue =
- ExportQueue.getInstance(RefExporter.QUEUE_ID, fc.getAppConfiguration());
- try (Transaction tx = fc.newTransaction()) {
- for (int i = 0; i < 1000; i++) {
- refExportQueue.add(tx, nk(i), new RefUpdates(ns(i + 10, i + 20), ns(new int[0])));
- }
-
- tx.commit();
- }
- }
-
- miniFluo.waitForObservers();
-
- Map<String, Set<String>> erefs = getExportedReferees();
- Map<String, Set<String>> expected = new HashMap<>();
-
- for (int i = 0; i < 1000; i++) {
- expected.computeIfAbsent(nk(i + 10), s -> new HashSet<>()).add(nk(i));
- expected.computeIfAbsent(nk(i + 20), s -> new HashSet<>()).add(nk(i));
- }
-
- assertEquals(expected, erefs);
- int prevNumExportCalls = getNumExportCalls();
- Assert.assertTrue(prevNumExportCalls > 10); // with small buffer there should be lots of exports
- // calls
-
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- ExportQueue<String, RefUpdates> refExportQueue =
- ExportQueue.getInstance(RefExporter.QUEUE_ID, fc.getAppConfiguration());
- try (Transaction tx = fc.newTransaction()) {
- for (int i = 0; i < 1000; i++) {
- refExportQueue.add(tx, nk(i), new RefUpdates(ns(i + 12), ns(i + 10)));
- }
-
- tx.commit();
- }
- }
-
- miniFluo.waitForObservers();
-
- erefs = getExportedReferees();
- expected = new HashMap<>();
-
- for (int i = 0; i < 1000; i++) {
- expected.computeIfAbsent(nk(i + 12), s -> new HashSet<>()).add(nk(i));
- expected.computeIfAbsent(nk(i + 20), s -> new HashSet<>()).add(nk(i));
- }
-
- assertEquals(expected, erefs);
- prevNumExportCalls = getNumExportCalls() - prevNumExportCalls;
- Assert.assertTrue(prevNumExportCalls > 10);
- }
-
- public void assertEquals(Map<String, Set<String>> expected, Map<String, Set<String>> actual) {
- if (!expected.equals(actual)) {
- System.out.println("*** diff ***");
- diff(expected, actual);
- Assert.fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportQueueIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportQueueIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportQueueIT.java
deleted file mode 100644
index baa979a..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportQueueIT.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.client.LoaderExecutor;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ExportQueueIT extends ExportTestBase {
-
- @Test
- public void testExport() {
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0999", "0005", "0002"));
- loader.execute(new DocumentLoader("0002", "0999", "0042"));
- loader.execute(new DocumentLoader("0005", "0999", "0042"));
- loader.execute(new DocumentLoader("0042", "0999"));
- }
-
- miniFluo.waitForObservers();
-
- Assert.assertEquals(ns("0002", "0005", "0042"), getExportedReferees("0999"));
- Assert.assertEquals(ns("0999"), getExportedReferees("0002"));
- Assert.assertEquals(ns("0999"), getExportedReferees("0005"));
- Assert.assertEquals(ns("0002", "0005"), getExportedReferees("0042"));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0999", "0005", "0042"));
- }
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0999", "0005"));
- }
-
- miniFluo.waitForObservers();
-
- Assert.assertEquals(ns("0002", "0005", "0042"), getExportedReferees("0999"));
- Assert.assertEquals(ns(new String[0]), getExportedReferees("0002"));
- Assert.assertEquals(ns("0999"), getExportedReferees("0005"));
- Assert.assertEquals(ns("0002", "0005"), getExportedReferees("0042"));
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0042", "0999", "0002", "0005"));
- loader.execute(new DocumentLoader("0005", "0002"));
- }
-
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- loader.execute(new DocumentLoader("0005", "0003"));
- }
-
- miniFluo.waitForObservers();
-
- Assert.assertEquals(ns("0002", "0042"), getExportedReferees("0999"));
- Assert.assertEquals(ns("0042"), getExportedReferees("0002"));
- Assert.assertEquals(ns("0005"), getExportedReferees("0003"));
- Assert.assertEquals(ns("0999", "0042"), getExportedReferees("0005"));
- Assert.assertEquals(ns("0002"), getExportedReferees("0042"));
-
- }
- }
-
- @Test
- public void exportStressTest() {
- FluoConfiguration config = new FluoConfiguration(miniFluo.getClientConfiguration());
- config.setLoaderQueueSize(100);
- config.setLoaderThreads(20);
-
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
-
- loadRandom(fc, 1000, 500);
-
- miniFluo.waitForObservers();
-
- diff(getFluoReferees(fc), getExportedReferees());
-
- assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
-
- loadRandom(fc, 1000, 500);
-
- miniFluo.waitForObservers();
-
- assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
-
- loadRandom(fc, 1000, 10000);
-
- miniFluo.waitForObservers();
-
- assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
-
- loadRandom(fc, 1000, 10000);
-
- miniFluo.waitForObservers();
-
- assertEquals(getFluoReferees(fc), getExportedReferees(), fc);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportTestBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportTestBase.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportTestBase.java
deleted file mode 100644
index 2584df4..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/ExportTestBase.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-
-import com.google.common.collect.Iterators;
-import org.apache.commons.io.FileUtils;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.client.LoaderExecutor;
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.ObserverConfiguration;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.api.mini.MiniFluo;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-public class ExportTestBase {
-
- private static Map<String, Map<String, RefInfo>> globalExports = new HashMap<>();
- private static int exportCalls = 0;
-
- protected static Set<String> getExportedReferees(String node) {
- synchronized (globalExports) {
- Set<String> ret = new HashSet<>();
-
- Map<String, RefInfo> referees = globalExports.get(node);
-
- if (referees == null) {
- return ret;
- }
-
- referees.forEach((k, v) -> {
- if (!v.deleted)
- ret.add(k);
- });
-
- return ret;
- }
- }
-
- protected static Map<String, Set<String>> getExportedReferees() {
- synchronized (globalExports) {
-
- Map<String, Set<String>> ret = new HashMap<>();
-
- for (String k : globalExports.keySet()) {
- Set<String> referees = getExportedReferees(k);
- if (referees.size() > 0) {
- ret.put(k, referees);
- }
- }
-
- return ret;
- }
- }
-
- protected static int getNumExportCalls() {
- synchronized (globalExports) {
- return exportCalls;
- }
- }
-
- public static class RefExporter extends Exporter<String, RefUpdates> {
-
- public static final String QUEUE_ID = "req";
-
- private void updateExports(String key, long seq, String addedRef, boolean deleted) {
- Map<String, RefInfo> referees = globalExports.computeIfAbsent(addedRef, k -> new HashMap<>());
- referees.compute(key, (k, v) -> (v == null || v.seq < seq) ? new RefInfo(seq, deleted) : v);
- }
-
- @Override
- protected void processExports(Iterator<SequencedExport<String, RefUpdates>> exportIterator) {
- ArrayList<SequencedExport<String, RefUpdates>> exportList = new ArrayList<>();
- Iterators.addAll(exportList, exportIterator);
-
- synchronized (globalExports) {
- exportCalls++;
-
- for (SequencedExport<String, RefUpdates> se : exportList) {
- for (String addedRef : se.getValue().getAddedRefs()) {
- updateExports(se.getKey(), se.getSequence(), addedRef, false);
- }
-
- for (String deletedRef : se.getValue().getDeletedRefs()) {
- updateExports(se.getKey(), se.getSequence(), deletedRef, true);
- }
- }
- }
- }
- }
-
- protected MiniFluo miniFluo;
-
- protected int getNumBuckets() {
- return 13;
- }
-
- protected Integer getBufferSize() {
- return null;
- }
-
- @Before
- public void setUpFluo() throws Exception {
- FileUtils.deleteQuietly(new File("target/mini"));
-
- FluoConfiguration props = new FluoConfiguration();
- props.setApplicationName("eqt");
- props.setWorkerThreads(20);
- props.setMiniDataDir("target/mini");
-
- ObserverConfiguration doc = new ObserverConfiguration(DocumentObserver.class.getName());
- props.addObserver(doc);
-
- SimpleSerializer.setSetserlializer(props, GsonSerializer.class);
-
- ExportQueue.Options exportQueueOpts =
- new ExportQueue.Options(RefExporter.QUEUE_ID, RefExporter.class, String.class,
- RefUpdates.class, getNumBuckets());
-
- if (getBufferSize() != null) {
- exportQueueOpts.setBufferSize(getBufferSize());
- }
-
- ExportQueue.configure(props, exportQueueOpts);
-
- miniFluo = FluoFactory.newMiniFluo(props);
-
- globalExports.clear();
- exportCalls = 0;
- }
-
- @After
- public void tearDownFluo() throws Exception {
- if (miniFluo != null) {
- miniFluo.close();
- }
- }
-
- protected static Set<String> ns(String... sa) {
- return new HashSet<>(Arrays.asList(sa));
- }
-
- protected static String nk(int i) {
- return String.format("%06d", i);
- }
-
- protected static Set<String> ns(int... ia) {
- HashSet<String> ret = new HashSet<>();
- for (int i : ia) {
- ret.add(nk(i));
- }
- return ret;
- }
-
- public void assertEquals(Map<String, Set<String>> expected, Map<String, Set<String>> actual,
- FluoClient fc) {
- if (!expected.equals(actual)) {
- System.out.println("*** diff ***");
- diff(expected, actual);
- System.out.println("*** fluo dump ***");
- dump(fc);
- System.out.println("*** map dump ***");
-
- Assert.fail();
- }
- }
-
- protected void loadRandom(FluoClient fc, int num, int maxDocId) {
- try (LoaderExecutor loader = fc.newLoaderExecutor()) {
- Random rand = new Random();
-
- for (int i = 0; i < num; i++) {
- String docid = String.format("%05d", rand.nextInt(maxDocId));
- String[] refs = new String[rand.nextInt(20) + 1];
- for (int j = 0; j < refs.length; j++) {
- refs[j] = String.format("%05d", rand.nextInt(maxDocId));
- }
-
- loader.execute(new DocumentLoader(docid, refs));
- }
- }
- }
-
- protected void diff(Map<String, Set<String>> fr, Map<String, Set<String>> er) {
- HashSet<String> allKeys = new HashSet<>(fr.keySet());
- allKeys.addAll(er.keySet());
-
- for (String k : allKeys) {
- Set<String> s1 = fr.getOrDefault(k, Collections.emptySet());
- Set<String> s2 = er.getOrDefault(k, Collections.emptySet());
-
- HashSet<String> sub1 = new HashSet<>(s1);
- sub1.removeAll(s2);
-
- HashSet<String> sub2 = new HashSet<>(s2);
- sub2.removeAll(s1);
-
- if (sub1.size() > 0 || sub2.size() > 0) {
- System.out.println(k + " " + sub1 + " " + sub2);
- }
-
- }
- }
-
- protected Map<String, Set<String>> getFluoReferees(FluoClient fc) {
- Map<String, Set<String>> fluoReferees = new HashMap<>();
-
- try (Snapshot snap = fc.newSnapshot()) {
- ScannerConfiguration scannerConfig = new ScannerConfiguration();
- scannerConfig.fetchColumn(Bytes.of("content"), Bytes.of("current"));
- scannerConfig.setSpan(Span.prefix("d:"));
- RowIterator scanner = snap.get(scannerConfig);
- while (scanner.hasNext()) {
- Entry<Bytes, ColumnIterator> row = scanner.next();
- ColumnIterator colIter = row.getValue();
-
- String docid = row.getKey().toString().substring(2);
-
- while (colIter.hasNext()) {
- Entry<Column, Bytes> entry = colIter.next();
-
- String[] refs = entry.getValue().toString().split(" ");
-
- for (String ref : refs) {
- if (ref.isEmpty())
- continue;
-
- fluoReferees.computeIfAbsent(ref, k -> new HashSet<>()).add(docid);
- }
- }
- }
- }
- return fluoReferees;
- }
-
- public static void dump(FluoClient fc) {
- try (Snapshot snap = fc.newSnapshot()) {
- RowIterator scanner = snap.get(new ScannerConfiguration());
- while (scanner.hasNext()) {
- Entry<Bytes, ColumnIterator> row = scanner.next();
- ColumnIterator colIter = row.getValue();
-
- while (colIter.hasNext()) {
- Entry<Column, Bytes> entry = colIter.next();
-
- System.out.println("row:[" + row.getKey() + "] col:[" + entry.getKey() + "] val:["
- + entry.getValue() + "]");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/GsonSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/GsonSerializer.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/GsonSerializer.java
deleted file mode 100644
index 61bb833..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/GsonSerializer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.nio.charset.StandardCharsets;
-
-import com.google.gson.Gson;
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-
-public class GsonSerializer implements SimpleSerializer {
-
- private Gson gson = new Gson();
-
- @Override
- public void init(SimpleConfiguration appConfig) {
-
- }
-
- @Override
- public <T> byte[] serialize(T obj) {
- return gson.toJson(obj).getBytes(StandardCharsets.UTF_8);
- }
-
- @Override
- public <T> T deserialize(byte[] serObj, Class<T> clazz) {
- return gson.fromJson(new String(serObj, StandardCharsets.UTF_8), clazz);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/OptionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/OptionsTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/OptionsTest.java
deleted file mode 100644
index 16e853b..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/OptionsTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.recipes.export.ExportQueue.Options;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class OptionsTest {
- @Test
- public void testExportQueueOptions() {
- FluoConfiguration conf = new FluoConfiguration();
-
- ExportQueue.configure(conf, new Options("Q1", "ET", "KT", "VT", 100));
- ExportQueue.configure(conf, new Options("Q2", "ET2", "KT2", "VT2", 200).setBucketsPerTablet(20)
- .setBufferSize(1000000));
-
- Options opts1 = new Options("Q1", conf.getAppConfiguration());
-
- Assert.assertEquals(opts1.exporterType, "ET");
- Assert.assertEquals(opts1.keyType, "KT");
- Assert.assertEquals(opts1.valueType, "VT");
- Assert.assertEquals(opts1.numBuckets, 100);
- Assert.assertEquals(opts1.bucketsPerTablet.intValue(), Options.DEFAULT_BUCKETS_PER_TABLET);
- Assert.assertEquals(opts1.bufferSize.intValue(), Options.DEFAULT_BUFFER_SIZE);
-
- Options opts2 = new Options("Q2", conf.getAppConfiguration());
-
- Assert.assertEquals(opts2.exporterType, "ET2");
- Assert.assertEquals(opts2.keyType, "KT2");
- Assert.assertEquals(opts2.valueType, "VT2");
- Assert.assertEquals(opts2.numBuckets, 200);
- Assert.assertEquals(opts2.bucketsPerTablet.intValue(), 20);
- Assert.assertEquals(opts2.bufferSize.intValue(), 1000000);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/RefInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/RefInfo.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/RefInfo.java
deleted file mode 100644
index abc518c..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/RefInfo.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-class RefInfo {
- long seq;
- boolean deleted;
-
- public RefInfo(long seq, boolean deleted) {
- this.seq = seq;
- this.deleted = deleted;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/export/RefUpdates.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/export/RefUpdates.java b/modules/core/src/test/java/org/apache/fluo/recipes/export/RefUpdates.java
deleted file mode 100644
index 31c61b7..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/export/RefUpdates.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Set;
-
-public class RefUpdates {
- private Set<String> addedRefs;
- private Set<String> deletedRefs;
-
- public RefUpdates() {}
-
- public RefUpdates(Set<String> addedRefs, Set<String> deletedRefs) {
- this.addedRefs = addedRefs;
- this.deletedRefs = deletedRefs;
- }
-
- public Set<String> getAddedRefs() {
- return addedRefs;
- }
-
- public Set<String> getDeletedRefs() {
- return deletedRefs;
- }
-
- @Override
- public String toString() {
- return "added:" + addedRefs + " deleted:" + deletedRefs;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/map/BigUpdateIT.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/map/BigUpdateIT.java b/modules/core/src/test/java/org/apache/fluo/recipes/map/BigUpdateIT.java
deleted file mode 100644
index d6409a6..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/map/BigUpdateIT.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
-import org.apache.commons.io.FileUtils;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.api.mini.MiniFluo;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-import org.apache.fluo.recipes.types.StringEncoder;
-import org.apache.fluo.recipes.types.TypeLayer;
-import org.apache.fluo.recipes.types.TypedSnapshot;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * This test configures a small buffer size and verifies that multiple passes are made to process
- * updates.
- */
-public class BigUpdateIT {
- private static final TypeLayer tl = new TypeLayer(new StringEncoder());
-
- private MiniFluo miniFluo;
-
- private CollisionFreeMap<String, Long> wcMap;
-
- static final String MAP_ID = "bu";
-
- public static class LongCombiner implements Combiner<String, Long> {
-
- @Override
- public Optional<Long> combine(String key, Iterator<Long> updates) {
- long[] count = new long[] {0};
- updates.forEachRemaining(l -> count[0] += l);
- return Optional.of(count[0]);
- }
- }
-
- static final Column DSCOL = new Column("debug", "sum");
-
- private static AtomicInteger globalUpdates = new AtomicInteger(0);
-
- public static class MyObserver extends UpdateObserver<String, Long> {
-
- @Override
- public void updatingValues(TransactionBase tx, Iterator<Update<String, Long>> updates) {
- TypedTransactionBase ttx = tl.wrap(tx);
-
- Map<String, Long> expectedOld = new HashMap<>();
-
-
- while (updates.hasNext()) {
- Update<String, Long> update = updates.next();
-
- if (update.getOldValue().isPresent()) {
- expectedOld.put("side:" + update.getKey(), update.getOldValue().get());
- }
-
- ttx.mutate().row("side:" + update.getKey()).col(DSCOL).set(update.getNewValue().get());
- }
-
- // get last values set to verify same as passed in old value
- Map<String, Long> actualOld =
- Maps.transformValues(
- ttx.get().rowsString(expectedOld.keySet()).columns(ImmutableSet.of(DSCOL))
- .toStringMap(), m -> m.get(DSCOL).toLong());
-
- MapDifference<String, Long> diff = Maps.difference(expectedOld, actualOld);
-
- Assert.assertTrue(diff.toString(), diff.areEqual());
-
- globalUpdates.incrementAndGet();
- }
- }
-
- @Before
- public void setUpFluo() throws Exception {
- FileUtils.deleteQuietly(new File("target/mini"));
-
- FluoConfiguration props = new FluoConfiguration();
- props.setApplicationName("eqt");
- props.setWorkerThreads(20);
- props.setMiniDataDir("target/mini");
-
- SimpleSerializer.setSetserlializer(props, TestSerializer.class);
-
- CollisionFreeMap.configure(props, new CollisionFreeMap.Options(MAP_ID, LongCombiner.class,
- MyObserver.class, String.class, Long.class, 2).setBufferSize(1 << 10));
-
- miniFluo = FluoFactory.newMiniFluo(props);
-
- wcMap = CollisionFreeMap.getInstance(MAP_ID, props.getAppConfiguration());
-
- globalUpdates.set(0);
- }
-
- @After
- public void tearDownFluo() throws Exception {
- if (miniFluo != null) {
- miniFluo.close();
- }
- }
-
- @Test
- public void testBigUpdates() {
- try (FluoClient fc = FluoFactory.newClient(miniFluo.getClientConfiguration())) {
- updateMany(fc);
-
- miniFluo.waitForObservers();
-
- int numUpdates = 0;
-
- try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
- checkUpdates(snap, 1, 1000);
- numUpdates = globalUpdates.get();
- // there are two buckets, expect update processing at least twice per bucket
- Assert.assertTrue(numUpdates >= 4);
- }
-
- updateMany(fc);
- updateMany(fc);
-
- miniFluo.waitForObservers();
-
- try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
- checkUpdates(snap, 3, 1000);
- numUpdates = globalUpdates.get() - numUpdates;
- Assert.assertTrue(numUpdates >= 4);
- }
-
- for (int i = 0; i < 10; i++) {
- updateMany(fc);
- }
-
- miniFluo.waitForObservers();
-
- try (TypedSnapshot snap = tl.wrap(fc.newSnapshot())) {
- checkUpdates(snap, 13, 1000);
- numUpdates = globalUpdates.get() - numUpdates;
- Assert.assertTrue(numUpdates >= 4);
- }
- }
- }
-
- private void checkUpdates(TypedSnapshot snap, long expectedVal, long expectedRows) {
- RowIterator iter = snap.get(new ScannerConfiguration().setSpan(Span.prefix("side:")));
-
- int row = 0;
-
- while (iter.hasNext()) {
- Entry<Bytes, ColumnIterator> entry = iter.next();
-
- Assert.assertEquals(String.format("side:%06d", row++), entry.getKey().toString());
-
- ColumnIterator colITer = entry.getValue();
- while (colITer.hasNext()) {
- Entry<Column, Bytes> entry2 = colITer.next();
- Assert.assertEquals(new Column("debug", "sum"), entry2.getKey());
- Assert.assertEquals("row : " + entry.getKey(), "" + expectedVal, entry2.getValue()
- .toString());
- }
- }
-
- Assert.assertEquals(expectedRows, row);
- }
-
- private void updateMany(FluoClient fc) {
- try (Transaction tx = fc.newTransaction()) {
- Map<String, Long> updates = new HashMap<>();
- for (int i = 0; i < 1000; i++) {
- updates.put(String.format("%06d", i), 1L);
- }
-
- wcMap.update(tx, updates);
- tx.commit();
- }
- }
-}
[09/10] incubator-fluo-recipes git commit: Added '@since 1.0.0'
annotations
Posted by kt...@apache.org.
Added '@since 1.0.0' annotations
* Also made minor whitespace changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/commit/083e4afb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/tree/083e4afb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/diff/083e4afb
Branch: refs/heads/master
Commit: 083e4afb1d6528c6cb8443ef24565f11187299a6
Parents: 594997a
Author: Mike Walch <mw...@gmail.com>
Authored: Fri Jul 15 12:26:07 2016 -0400
Committer: Mike Walch <mw...@gmail.com>
Committed: Fri Jul 15 12:38:05 2016 -0400
----------------------------------------------------------------------
.../apache/fluo/recipes/accumulo/cmds/CompactTransient.java | 9 +++------
.../apache/fluo/recipes/accumulo/cmds/OptimizeTable.java | 4 +++-
.../apache/fluo/recipes/accumulo/export/AccumuloExport.java | 1 +
.../fluo/recipes/accumulo/export/AccumuloExporter.java | 1 +
.../fluo/recipes/accumulo/export/DifferenceExport.java | 1 +
.../fluo/recipes/accumulo/export/ReplicationExport.java | 1 +
.../org/apache/fluo/recipes/accumulo/export/TableInfo.java | 3 +++
.../apache/fluo/recipes/accumulo/ops/TableOperations.java | 3 ++-
.../java/org/apache/fluo/recipes/core/common/Pirtos.java | 3 ++-
.../java/org/apache/fluo/recipes/core/common/RowRange.java | 3 +++
.../apache/fluo/recipes/core/common/TransientRegistry.java | 3 ++-
.../java/org/apache/fluo/recipes/core/data/RowHasher.java | 2 ++
.../java/org/apache/fluo/recipes/core/export/Export.java | 3 +++
.../org/apache/fluo/recipes/core/export/ExportObserver.java | 3 +++
.../org/apache/fluo/recipes/core/export/ExportQueue.java | 3 +++
.../java/org/apache/fluo/recipes/core/export/Exporter.java | 3 +++
.../apache/fluo/recipes/core/export/SequencedExport.java | 3 +++
.../org/apache/fluo/recipes/core/map/CollisionFreeMap.java | 4 ++--
.../fluo/recipes/core/map/CollisionFreeMapObserver.java | 3 ++-
.../java/org/apache/fluo/recipes/core/map/Combiner.java | 5 ++++-
.../main/java/org/apache/fluo/recipes/core/map/Update.java | 3 +++
.../org/apache/fluo/recipes/core/map/UpdateObserver.java | 3 ++-
.../fluo/recipes/core/serialization/SimpleSerializer.java | 3 +++
.../org/apache/fluo/recipes/core/transaction/LogEntry.java | 2 ++
.../fluo/recipes/core/transaction/RecordingTransaction.java | 2 ++
.../recipes/core/transaction/RecordingTransactionBase.java | 2 ++
.../org/apache/fluo/recipes/core/transaction/TxLog.java | 2 ++
.../org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java | 3 +++
.../apache/fluo/recipes/spark/AccumuloRangePartitioner.java | 3 +++
.../java/org/apache/fluo/recipes/spark/FluoSparkHelper.java | 2 ++
.../org/apache/fluo/recipes/spark/FluoSparkTestUtil.java | 2 ++
.../org/apache/fluo/recipes/test/AccumuloExportITBase.java | 2 ++
.../java/org/apache/fluo/recipes/test/FluoITHelper.java | 2 ++
33 files changed, 77 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
index 54e05bc..3ec97c2 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
@@ -32,13 +32,15 @@ import org.apache.fluo.recipes.core.common.TransientRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * @since 1.0.0
+ */
public class CompactTransient {
// when run with fluo exec command, the applications fluo config will be injected
@Inject
private static FluoConfiguration fluoConfig;
-
private static ScheduledExecutorService schedExecutor;
private static Logger log = LoggerFactory.getLogger(CompactTransient.class);
@@ -77,12 +79,7 @@ public class CompactTransient {
} else {
log.info("Compacted {} in {}ms", transientRange, t2 - t1);
}
-
-
}
-
-
-
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
index bf17cfd..2a56cfb 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
@@ -21,6 +21,9 @@ import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
import org.apache.fluo.recipes.core.common.Pirtos;
+/**
+ * @since 1.0.0
+ */
public class OptimizeTable {
// when run with fluo exec command, the applications fluo config will be injected
@@ -33,7 +36,6 @@ public class OptimizeTable {
System.exit(-1);
}
-
TableOperations.optimizeTable(fluoConfig, Pirtos.getConfiguredOptimizations(fluoConfig));
System.out.println("Finished optimizing table");
}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExport.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExport.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExport.java
index f6d7a51..9b31b7c 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExport.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExport.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.data.Mutation;
* Implemented by users to export data to Accumulo.
*
* @param <K> Export queue key type
+ * @since 1.0.0
*/
public interface AccumuloExport<K> {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
index 1788403..7e48d12 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
@@ -29,6 +29,7 @@ import org.apache.fluo.recipes.core.export.SequencedExport;
* An {@link Exporter} that takes {@link AccumuloExport} objects and writes mutations to Accumulo
*
* @param <K> Export queue key type
+ * @since 1.0.0
*/
public class AccumuloExporter<K> extends Exporter<K, AccumuloExport<K>> {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/DifferenceExport.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/DifferenceExport.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/DifferenceExport.java
index 015cc98..d6fefa7 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/DifferenceExport.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/DifferenceExport.java
@@ -44,6 +44,7 @@ import org.apache.fluo.api.data.RowColumn;
*
* @param <K> Export queue key type
* @param <V> Type of export value object used to generate data
+ * @since 1.0.0
*/
public abstract class DifferenceExport<K, V> implements AccumuloExport<K> {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
index 9a3b196..9276c6d 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
@@ -33,6 +33,7 @@ import org.apache.fluo.recipes.core.transaction.TxLog;
* TxLog
*
* @param <K> Type of export queue key
+ * @since 1.0.0
*/
public class ReplicationExport<K> implements AccumuloExport<K> {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/TableInfo.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/TableInfo.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/TableInfo.java
index 04f7c05..9da6773 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/TableInfo.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/TableInfo.java
@@ -15,6 +15,9 @@
package org.apache.fluo.recipes.accumulo.export;
+/**
+ * @since 1.0.0
+ */
public class TableInfo {
String instanceName;
String zookeepers;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
index 62abd7e..3cc418c 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
@@ -37,8 +37,9 @@ import org.slf4j.LoggerFactory;
/**
* Utility methods for operating on the Fluo table used by recipes.
+ *
+ * @since 1.0.0
*/
-
public class TableOperations {
private static final String RGB_CLASS =
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
index 5488e29..8b36472 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
@@ -30,8 +30,9 @@ import org.apache.fluo.recipes.core.map.CollisionFreeMap;
/**
* Post initialization recommended table optimizations.
+ *
+ * @since 1.0.0
*/
-
public class Pirtos {
private List<Bytes> splits = new ArrayList<>();
private String tabletGroupingRegex = "";
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
index 913357b..361dd6e 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
@@ -19,6 +19,9 @@ import java.util.Objects;
import org.apache.fluo.api.data.Bytes;
+/**
+ * @since 1.0.0
+ */
public class RowRange {
private final Bytes start;
private final Bytes end;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
index 9533a7a..dc3922f 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
@@ -29,8 +29,9 @@ import org.apache.fluo.api.data.Bytes;
/**
* This class offers a standard way to register transient ranges. The project level documentation
* provides a comprehensive overview.
+ *
+ * @since 1.0.0
*/
-
public class TransientRegistry {
private SimpleConfiguration appConfig;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
index e09bed5..ace7e7e 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
@@ -39,6 +39,8 @@ import org.apache.fluo.recipes.core.common.Pirtos;
*
* <p>
* The project documentation has more information.
+ *
+ * @since 1.0.0
*/
public class RowHasher {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
index 4f14f65..5325d43 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
@@ -17,6 +17,9 @@ package org.apache.fluo.recipes.core.export;
import java.util.Objects;
+/**
+ * @since 1.0.0
+ */
public class Export<K, V> {
private final K key;
private final V value;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
index 940575b..e09a376 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
@@ -25,6 +25,9 @@ import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.observer.AbstractObserver;
import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+/**
+ * @since 1.0.0
+ */
public class ExportObserver<K, V> extends AbstractObserver {
private static class MemLimitIterator implements Iterator<ExportEntry> {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
index f013872..ac04f80 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
@@ -35,6 +35,9 @@ import org.apache.fluo.recipes.core.common.RowRange;
import org.apache.fluo.recipes.core.common.TransientRegistry;
import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+/**
+ * @since 1.0.0
+ */
public class ExportQueue<K, V> {
private static final String RANGE_BEGIN = "#";
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
index 529d5f3..f00cfb0 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
@@ -19,6 +19,9 @@ import java.util.Iterator;
import org.apache.fluo.api.observer.Observer.Context;
+/**
+ * @since 1.0.0
+ */
public abstract class Exporter<K, V> {
public void init(String queueId, Context observerContext) throws Exception {}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
index ef1cfe2..e944092 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
@@ -15,6 +15,9 @@
package org.apache.fluo.recipes.core.export;
+/**
+ * @since 1.0.0
+ */
public class SequencedExport<K, V> extends Export<K, V> {
private final long seq;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
index 61fec47..6cbc658 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
@@ -55,6 +55,8 @@ import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
/**
* See the project level documentation for information about this recipe.
+ *
+ * @since 1.0.0
*/
public class CollisionFreeMap<K, V> {
@@ -374,7 +376,6 @@ public class CollisionFreeMap<K, V> {
}
}
-
public static <K2, V2> CollisionFreeMap<K2, V2> getInstance(String mapId,
SimpleConfiguration appConf) {
Options opts = new Options(mapId, appConf);
@@ -398,7 +399,6 @@ public class CollisionFreeMap<K, V> {
return new Initializer<>(mapId, numBuckets, serializer);
}
-
/**
* @see CollisionFreeMap#getInitializer(String, int, SimpleSerializer)
*/
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
index 96890af..581bb10 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
@@ -23,8 +23,9 @@ import org.apache.fluo.api.observer.AbstractObserver;
/**
* This class is configured for use by CollisionFreeMap.configure(FluoConfiguration,
* CollisionFreeMap.Options) . This class should never have to be used directly.
+ *
+ * @since 1.0.0
*/
-
public class CollisionFreeMapObserver extends AbstractObserver {
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
index c9d468b..4d3ee77 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
@@ -18,7 +18,11 @@ package org.apache.fluo.recipes.core.map;
import java.util.Iterator;
import java.util.Optional;
+/**
+ * @since 1.0.0
+ */
public interface Combiner<K, V> {
+
/**
* This function is called to combine the current value of a key with updates that were queued for
* the key. See the collision free map project level documentation for more information.
@@ -26,6 +30,5 @@ public interface Combiner<K, V> {
* @return Then new value for the key. Returning Optional.absent() will cause the key to be
* deleted.
*/
-
Optional<V> combine(K key, Iterator<V> updates);
}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
index 10e718e..1965111 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
@@ -17,6 +17,9 @@ package org.apache.fluo.recipes.core.map;
import java.util.Optional;
+/**
+ * @since 1.0.0
+ */
public class Update<K, V> {
private final K key;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
index 2e48451..b00e1b3 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
@@ -23,8 +23,9 @@ import org.apache.fluo.api.observer.Observer.Context;
/**
* A {@link CollisionFreeMap} calls this to allow additional processing to be done when key values
* are updated. See the project level documentation for more information.
+ *
+ * @since 1.0.0
*/
-
public abstract class UpdateObserver<K, V> {
public void init(String mapId, Context observerContext) throws Exception {}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
index 589b9cc..4efed5f 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
@@ -18,6 +18,9 @@ package org.apache.fluo.recipes.core.serialization;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
+/**
+ * @since 1.0.0
+ */
public interface SimpleSerializer {
/**
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
index fa25b63..6023e4a 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
@@ -23,6 +23,8 @@ import org.apache.fluo.api.data.Column;
/**
* Logs an operation (i.e GET, SET, or DELETE) in a Transaction. Multiple LogEntry objects make up a
* {@link TxLog}.
+ *
+ * @since 1.0.0
*/
public class LogEntry {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
index 8fb98d3..06a1e74 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
@@ -23,6 +23,8 @@ import org.apache.fluo.api.exceptions.CommitException;
/**
* An implementation of {@link Transaction} that logs all transactions operations (GET, SET, or
* DELETE) in a {@link TxLog} that can be used for exports
+ *
+ * @since 1.0.0
*/
public class RecordingTransaction extends RecordingTransactionBase implements Transaction {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
index 6303122..a29cf88 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
@@ -33,6 +33,8 @@ import org.apache.fluo.api.iterator.RowIterator;
/**
* An implementation of {@link TransactionBase} that logs all transactions operations (GET, SET, or
* DELETE) in a {@link TxLog} that can be used for exports
+ *
+ * @since 1.0.0
*/
public class RecordingTransactionBase implements TransactionBase {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
index caa3c4d..70bd706 100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
@@ -27,6 +27,8 @@ import org.apache.fluo.api.data.RowColumn;
/**
* Contains list of operations (GET, SET, DELETE) performed during a {@link RecordingTransaction}
+ *
+ * @since 1.0.0
*/
public class TxLog {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
----------------------------------------------------------------------
diff --git a/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java b/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
index 851f045..5c10d85 100644
--- a/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
+++ b/modules/kryo/src/main/java/org/apache/fluo/recipes/kryo/KryoSimplerSerializer.java
@@ -32,6 +32,9 @@ import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+/***
+ * @since 1.0.0
+ */
public class KryoSimplerSerializer implements SimpleSerializer, Serializable {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/spark/src/main/java/org/apache/fluo/recipes/spark/AccumuloRangePartitioner.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/AccumuloRangePartitioner.java b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/AccumuloRangePartitioner.java
index 7c715de..9e662e3 100644
--- a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/AccumuloRangePartitioner.java
+++ b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/AccumuloRangePartitioner.java
@@ -25,6 +25,9 @@ import org.apache.fluo.api.data.RowColumn;
import org.apache.hadoop.io.Text;
import org.apache.spark.Partitioner;
+/**
+ * @since 1.0.0
+ */
public class AccumuloRangePartitioner extends Partitioner {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkHelper.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkHelper.java b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkHelper.java
index 029ff71..e865888 100644
--- a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkHelper.java
+++ b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkHelper.java
@@ -54,6 +54,8 @@ import scala.Tuple2;
/**
* Helper methods for using Fluo with Spark
+ *
+ * @since 1.0.0
*/
public class FluoSparkHelper {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkTestUtil.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkTestUtil.java b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkTestUtil.java
index 10d3818..163bb8f 100644
--- a/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkTestUtil.java
+++ b/modules/spark/src/main/java/org/apache/fluo/recipes/spark/FluoSparkTestUtil.java
@@ -21,6 +21,8 @@ import org.apache.spark.api.java.JavaSparkContext;
/**
* Utility code for Fluo/Spark testing
+ *
+ * @since 1.0.0
*/
public class FluoSparkTestUtil {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
----------------------------------------------------------------------
diff --git a/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java b/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
index 2d1bbb4..c1adc3b 100644
--- a/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
+++ b/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
@@ -90,6 +90,8 @@ import org.junit.BeforeClass;
* }
* </code>
* </pre>
+ *
+ * @since 1.0.0
*/
public class AccumuloExportITBase {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/083e4afb/modules/test/src/main/java/org/apache/fluo/recipes/test/FluoITHelper.java
----------------------------------------------------------------------
diff --git a/modules/test/src/main/java/org/apache/fluo/recipes/test/FluoITHelper.java b/modules/test/src/main/java/org/apache/fluo/recipes/test/FluoITHelper.java
index 229d562..92d7eaf 100644
--- a/modules/test/src/main/java/org/apache/fluo/recipes/test/FluoITHelper.java
+++ b/modules/test/src/main/java/org/apache/fluo/recipes/test/FluoITHelper.java
@@ -45,6 +45,8 @@ import org.slf4j.LoggerFactory;
/**
* Helper for creating integration tests that connect to a MiniFluo/MiniAccumuloCluster instance.
+ *
+ * @since 1.0.0
*/
public class FluoITHelper {
[10/10] incubator-fluo-recipes git commit: Merge remote-tracking
branch 'mike/package-refactor'
Posted by kt...@apache.org.
Merge remote-tracking branch 'mike/package-refactor'
Conflicts:
docs/row-hasher.md
modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
modules/core/src/main/java/org/apache/fluo/recipes/common/Pirtos.java
modules/core/src/main/java/org/apache/fluo/recipes/common/TableOptimizations.java
modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/commit/22354d0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/tree/22354d0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/diff/22354d0f
Branch: refs/heads/master
Commit: 22354d0f7f03f532ec093b7ece61b5ae9fb070e9
Parents: a8b85f3 083e4af
Author: Keith Turner <kt...@apache.org>
Authored: Fri Jul 15 18:01:55 2016 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Jul 15 18:01:55 2016 -0400
----------------------------------------------------------------------
docs/export-queue.md | 2 +-
docs/row-hasher.md | 4 +-
docs/serialization.md | 2 +-
docs/transient.md | 2 +-
.../recipes/accumulo/cmds/CompactTransient.java | 13 +-
.../recipes/accumulo/cmds/OptimizeTable.java | 6 +-
.../recipes/accumulo/export/AccumuloExport.java | 1 +
.../accumulo/export/AccumuloExporter.java | 5 +-
.../accumulo/export/DifferenceExport.java | 1 +
.../accumulo/export/ReplicationExport.java | 5 +-
.../accumulo/export/SharedBatchWriter.java | 14 +-
.../fluo/recipes/accumulo/export/TableInfo.java | 3 +
.../recipes/accumulo/ops/TableOperations.java | 9 +-
.../apache/fluo/recipes/common/RowRange.java | 82 ---
.../fluo/recipes/common/TableOptimizations.java | 82 ---
.../fluo/recipes/common/TransientRegistry.java | 79 ---
.../fluo/recipes/core/common/RowRange.java | 85 +++
.../recipes/core/common/TableOptimizations.java | 84 +++
.../recipes/core/common/TransientRegistry.java | 80 +++
.../fluo/recipes/core/data/RowHasher.java | 137 ++++
.../apache/fluo/recipes/core/export/Export.java | 41 ++
.../fluo/recipes/core/export/ExportBucket.java | 203 ++++++
.../fluo/recipes/core/export/ExportEntry.java | 22 +
.../recipes/core/export/ExportObserver.java | 143 ++++
.../fluo/recipes/core/export/ExportQueue.java | 277 ++++++++
.../fluo/recipes/core/export/Exporter.java | 67 ++
.../recipes/core/export/SequencedExport.java | 32 +
.../fluo/recipes/core/impl/BucketUtil.java | 24 +
.../fluo/recipes/core/map/CollisionFreeMap.java | 657 +++++++++++++++++++
.../core/map/CollisionFreeMapObserver.java | 54 ++
.../apache/fluo/recipes/core/map/Combiner.java | 34 +
.../recipes/core/map/NullUpdateObserver.java | 25 +
.../apache/fluo/recipes/core/map/Update.java | 46 ++
.../fluo/recipes/core/map/UpdateObserver.java | 35 +
.../core/serialization/SimpleSerializer.java | 59 ++
.../fluo/recipes/core/transaction/LogEntry.java | 116 ++++
.../core/transaction/RecordingTransaction.java | 66 ++
.../transaction/RecordingTransactionBase.java | 252 +++++++
.../fluo/recipes/core/transaction/TxLog.java | 81 +++
.../apache/fluo/recipes/core/types/Encoder.java | 86 +++
.../fluo/recipes/core/types/StringEncoder.java | 86 +++
.../fluo/recipes/core/types/TypeLayer.java | 488 ++++++++++++++
.../fluo/recipes/core/types/TypedLoader.java | 45 ++
.../fluo/recipes/core/types/TypedObserver.java | 46 ++
.../fluo/recipes/core/types/TypedSnapshot.java | 38 ++
.../recipes/core/types/TypedSnapshotBase.java | 555 ++++++++++++++++
.../recipes/core/types/TypedTransaction.java | 46 ++
.../core/types/TypedTransactionBase.java | 278 ++++++++
.../org/apache/fluo/recipes/data/RowHasher.java | 135 ----
.../org/apache/fluo/recipes/export/Export.java | 38 --
.../fluo/recipes/export/ExportBucket.java | 203 ------
.../apache/fluo/recipes/export/ExportEntry.java | 22 -
.../fluo/recipes/export/ExportObserver.java | 140 ----
.../apache/fluo/recipes/export/ExportQueue.java | 274 --------
.../apache/fluo/recipes/export/Exporter.java | 64 --
.../fluo/recipes/export/SequencedExport.java | 29 -
.../apache/fluo/recipes/impl/BucketUtil.java | 24 -
.../fluo/recipes/map/CollisionFreeMap.java | 657 -------------------
.../recipes/map/CollisionFreeMapObserver.java | 53 --
.../org/apache/fluo/recipes/map/Combiner.java | 31 -
.../fluo/recipes/map/NullUpdateObserver.java | 25 -
.../org/apache/fluo/recipes/map/Update.java | 43 --
.../apache/fluo/recipes/map/UpdateObserver.java | 34 -
.../recipes/serialization/SimpleSerializer.java | 56 --
.../fluo/recipes/transaction/LogEntry.java | 114 ----
.../transaction/RecordingTransaction.java | 64 --
.../transaction/RecordingTransactionBase.java | 250 -------
.../apache/fluo/recipes/transaction/TxLog.java | 79 ---
.../org/apache/fluo/recipes/types/Encoder.java | 86 ---
.../fluo/recipes/types/StringEncoder.java | 86 ---
.../apache/fluo/recipes/types/TypeLayer.java | 488 --------------
.../apache/fluo/recipes/types/TypedLoader.java | 45 --
.../fluo/recipes/types/TypedObserver.java | 46 --
.../fluo/recipes/types/TypedSnapshot.java | 38 --
.../fluo/recipes/types/TypedSnapshotBase.java | 555 ----------------
.../fluo/recipes/types/TypedTransaction.java | 46 --
.../recipes/types/TypedTransactionBase.java | 278 --------
.../fluo/recipes/common/TestGrouping.java | 95 ---
.../recipes/common/TransientRegistryTest.java | 48 --
.../fluo/recipes/core/common/TestGrouping.java | 93 +++
.../core/common/TransientRegistryTest.java | 48 ++
.../fluo/recipes/core/data/RowHasherTest.java | 62 ++
.../recipes/core/export/DocumentLoader.java | 36 +
.../recipes/core/export/DocumentObserver.java | 63 ++
.../recipes/core/export/ExportBufferIT.java | 106 +++
.../fluo/recipes/core/export/ExportQueueIT.java | 114 ++++
.../recipes/core/export/ExportTestBase.java | 286 ++++++++
.../recipes/core/export/GsonSerializer.java | 42 ++
.../fluo/recipes/core/export/OptionsTest.java | 51 ++
.../fluo/recipes/core/export/RefInfo.java | 26 +
.../fluo/recipes/core/export/RefUpdates.java | 43 ++
.../fluo/recipes/core/map/BigUpdateIT.java | 214 ++++++
.../recipes/core/map/CollisionFreeMapIT.java | 361 ++++++++++
.../fluo/recipes/core/map/DocumentLoader.java | 35 +
.../fluo/recipes/core/map/DocumentObserver.java | 89 +++
.../fluo/recipes/core/map/OptionsTest.java | 51 ++
.../fluo/recipes/core/map/SplitsTest.java | 76 +++
.../fluo/recipes/core/map/TestSerializer.java | 45 ++
.../recipes/core/map/WordCountCombiner.java | 36 +
.../recipes/core/map/WordCountObserver.java | 47 ++
.../transaction/RecordingTransactionTest.java | 227 +++++++
.../fluo/recipes/core/types/MockSnapshot.java | 30 +
.../recipes/core/types/MockSnapshotBase.java | 202 ++++++
.../recipes/core/types/MockTransaction.java | 36 +
.../recipes/core/types/MockTransactionBase.java | 90 +++
.../fluo/recipes/core/types/TypeLayerTest.java | 494 ++++++++++++++
.../apache/fluo/recipes/data/RowHasherTest.java | 62 --
.../fluo/recipes/export/DocumentLoader.java | 36 -
.../fluo/recipes/export/DocumentObserver.java | 63 --
.../fluo/recipes/export/ExportBufferIT.java | 106 ---
.../fluo/recipes/export/ExportQueueIT.java | 114 ----
.../fluo/recipes/export/ExportTestBase.java | 286 --------
.../fluo/recipes/export/GsonSerializer.java | 42 --
.../apache/fluo/recipes/export/OptionsTest.java | 51 --
.../org/apache/fluo/recipes/export/RefInfo.java | 26 -
.../apache/fluo/recipes/export/RefUpdates.java | 43 --
.../apache/fluo/recipes/map/BigUpdateIT.java | 214 ------
.../fluo/recipes/map/CollisionFreeMapIT.java | 361 ----------
.../apache/fluo/recipes/map/DocumentLoader.java | 35 -
.../fluo/recipes/map/DocumentObserver.java | 89 ---
.../apache/fluo/recipes/map/OptionsTest.java | 51 --
.../org/apache/fluo/recipes/map/SplitsTest.java | 76 ---
.../apache/fluo/recipes/map/TestSerializer.java | 45 --
.../fluo/recipes/map/WordCountCombiner.java | 36 -
.../fluo/recipes/map/WordCountObserver.java | 47 --
.../transaction/RecordingTransactionTest.java | 227 -------
.../apache/fluo/recipes/types/MockSnapshot.java | 30 -
.../fluo/recipes/types/MockSnapshotBase.java | 202 ------
.../fluo/recipes/types/MockTransaction.java | 36 -
.../fluo/recipes/types/MockTransactionBase.java | 90 ---
.../fluo/recipes/types/TypeLayerTest.java | 494 --------------
.../recipes/kryo/KryoSimplerSerializer.java | 5 +-
.../serialization/KryoSimpleSerializerTest.java | 45 ++
.../serialization/KryoSimpleSerializerTest.java | 45 --
.../recipes/spark/AccumuloRangePartitioner.java | 3 +
.../fluo/recipes/spark/FluoSparkHelper.java | 2 +
.../fluo/recipes/spark/FluoSparkTestUtil.java | 2 +
.../fluo/recipes/test/AccumuloExportITBase.java | 4 +-
.../apache/fluo/recipes/test/FluoITHelper.java | 2 +
.../recipes/test/export/AccumuloExporterIT.java | 2 +-
.../test/export/AccumuloReplicatorIT.java | 10 +-
141 files changed, 7395 insertions(+), 7334 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/docs/row-hasher.md
----------------------------------------------------------------------
diff --cc docs/row-hasher.md
index 78aab23,8db8af6..fdd8218
--- a/docs/row-hasher.md
+++ b/docs/row-hasher.md
@@@ -31,8 -31,8 +31,8 @@@ balancing of the prefix
```java
import org.apache.fluo.api.data.Bytes;
- import org.apache.fluo.recipes.common.TableOptimizations;
- import org.apache.fluo.recipes.data.RowHasher;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ import org.apache.fluo.recipes.core.data.RowHasher;
public class RowHasherExample {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
----------------------------------------------------------------------
diff --cc modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
index 7910bdb,2a56cfb..92651bd
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
@@@ -19,8 -19,11 +19,11 @@@ import javax.inject.Inject
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
- import org.apache.fluo.recipes.common.TableOptimizations;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ /**
+ * @since 1.0.0
+ */
public class OptimizeTable {
// when run with fluo exec command, the applications fluo config will be injected
@@@ -33,9 -36,7 +36,8 @@@
System.exit(-1);
}
-
- TableOperations.optimizeTable(fluoConfig, Pirtos.getConfiguredOptimizations(fluoConfig));
+ TableOperations.optimizeTable(fluoConfig,
+ TableOptimizations.getConfiguredOptimizations(fluoConfig));
System.out.println("Finished optimizing table");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
----------------------------------------------------------------------
diff --cc modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
index c4fd07f,3cc418c..e433317
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
@@@ -28,9 -28,9 +28,9 @@@ import org.apache.fluo.api.client.FluoF
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
import org.apache.fluo.api.data.Bytes;
- import org.apache.fluo.recipes.common.TableOptimizations;
- import org.apache.fluo.recipes.common.RowRange;
- import org.apache.fluo.recipes.common.TransientRegistry;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ import org.apache.fluo.recipes.core.common.RowRange;
+ import org.apache.fluo.recipes.core.common.TransientRegistry;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TableOptimizations.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/fluo/recipes/core/common/TableOptimizations.java
index 0000000,0000000..4657366
new file mode 100644
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TableOptimizations.java
@@@ -1,0 -1,0 +1,84 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
++ * agreements. See the NOTICE file distributed with this work for additional information regarding
++ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance with the License. You may obtain a
++ * copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software distributed under the License
++ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
++ * or implied. See the License for the specific language governing permissions and limitations under
++ * the License.
++ */
++
++package org.apache.fluo.recipes.core.common;
++
++import java.util.ArrayList;
++import java.util.Collections;
++import java.util.List;
++import java.util.Objects;
++
++import org.apache.fluo.api.client.FluoClient;
++import org.apache.fluo.api.client.FluoFactory;
++import org.apache.fluo.api.config.FluoConfiguration;
++import org.apache.fluo.api.config.SimpleConfiguration;
++import org.apache.fluo.api.data.Bytes;
++import org.apache.fluo.recipes.core.export.ExportQueue;
++import org.apache.fluo.recipes.core.map.CollisionFreeMap;
++
++/**
++ * Post initialization recommended table optimizations.
++ *
++ * @since 1.0.0
++ */
++public class TableOptimizations {
++ private List<Bytes> splits = new ArrayList<>();
++ private String tabletGroupingRegex = "";
++
++ public void setSplits(List<Bytes> splits) {
++ this.splits.clear();
++ this.splits.addAll(splits);
++ }
++
++ /**
++ * @return A recommended set of splits points to add to a Fluo table after initialization.
++ */
++ public List<Bytes> getSplits() {
++ return Collections.unmodifiableList(splits);
++ }
++
++ public void setTabletGroupingRegex(String tgr) {
++ Objects.requireNonNull(tgr);
++ this.tabletGroupingRegex = tgr;
++ }
++
++ public String getTabletGroupingRegex() {
++ return "(" + tabletGroupingRegex + ").*";
++ }
++
++ public void merge(TableOptimizations other) {
++ splits.addAll(other.splits);
++ if (tabletGroupingRegex.length() > 0 && other.tabletGroupingRegex.length() > 0) {
++ tabletGroupingRegex += "|" + other.tabletGroupingRegex;
++ } else {
++ tabletGroupingRegex += other.tabletGroupingRegex;
++ }
++ }
++
++ /**
++ * A utility method to get table optimizations for all configured recipes.
++ */
++ public static TableOptimizations getConfiguredOptimizations(FluoConfiguration fluoConfig) {
++ try (FluoClient client = FluoFactory.newClient(fluoConfig)) {
++ SimpleConfiguration appConfig = client.getAppConfiguration();
++ TableOptimizations tableOptim = new TableOptimizations();
++
++ tableOptim.merge(ExportQueue.getTableOptimizations(appConfig));
++ tableOptim.merge(CollisionFreeMap.getTableOptimizations(appConfig));
++
++ return tableOptim;
++ }
++ }
++}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
index 0000000,ace7e7e..e40ce9b
mode 000000,100644..100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
@@@ -1,0 -1,137 +1,137 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ package org.apache.fluo.recipes.core.data;
+
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.regex.Pattern;
+
+ import com.google.common.base.Preconditions;
+ import com.google.common.base.Strings;
+ import com.google.common.hash.Hashing;
+ import org.apache.fluo.api.data.Bytes;
+ import org.apache.fluo.api.data.BytesBuilder;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+
+ /**
+ * This recipe provides code to help add a hash of the row as a prefix of the row. Using this recipe
+ * rows are structured like the following.
+ *
+ * <p>
+ * {@code <prefix>:<fixed len row hash>:<user row>}
+ *
+ * <p>
+ * The recipe also provides code the help generate split points and configure balancing of the
+ * prefix.
+ *
+ * <p>
+ * The project documentation has more information.
+ *
+ * @since 1.0.0
+ */
+ public class RowHasher {
+
+ private static final int HASH_LEN = 4;
+
- public Pirtos getTableOptimizations(int numTablets) {
++ public TableOptimizations getTableOptimizations(int numTablets) {
+
+ List<Bytes> splits = new ArrayList<>(numTablets - 1);
+
+ int numSplits = numTablets - 1;
+ int distance = (((int) Math.pow(Character.MAX_RADIX, HASH_LEN) - 1) / numTablets) + 1;
+ int split = distance;
+ for (int i = 0; i < numSplits; i++) {
+ splits.add(Bytes.of(prefix
+ + Strings.padStart(Integer.toString(split, Character.MAX_RADIX), HASH_LEN, '0')));
+ split += distance;
+ }
+
+ splits.add(Bytes.of(prefix + "~"));
+
+
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
- pirtos.setTabletGroupingRegex(Pattern.quote(prefix.toString()));
++ TableOptimizations tableOptim = new TableOptimizations();
++ tableOptim.setSplits(splits);
++ tableOptim.setTabletGroupingRegex(Pattern.quote(prefix.toString()));
+
- return pirtos;
++ return tableOptim;
+ }
+
+
+ private Bytes prefix;
+
+ public RowHasher(String prefix) {
+ this.prefix = Bytes.of(prefix + ":");
+ }
+
+ /**
+ * @return Returns input with prefix and hash of input prepended.
+ */
+ public Bytes addHash(String row) {
+ return addHash(Bytes.of(row));
+ }
+
+ /**
+ * @return Returns input with prefix and hash of input prepended.
+ */
+ public Bytes addHash(Bytes row) {
+ BytesBuilder builder = Bytes.newBuilder(prefix.length() + 5 + row.length());
+ builder.append(prefix);
+ builder.append(genHash(row));
+ builder.append(":");
+ builder.append(row);
+ return builder.toBytes();
+ }
+
+ private boolean hasHash(Bytes row) {
+ for (int i = prefix.length(); i < prefix.length() + HASH_LEN; i++) {
+ byte b = row.byteAt(i);
+ boolean isAlphaNum = (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9');
+ if (!isAlphaNum) {
+ return false;
+ }
+ }
+
+ if (row.byteAt(prefix.length() - 1) != ':' || row.byteAt(prefix.length() + HASH_LEN) != ':') {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * @return Returns input with prefix and hash stripped from beginning.
+ */
+ public Bytes removeHash(Bytes row) {
+ Preconditions.checkArgument(row.length() >= prefix.length() + 5,
+ "Row is shorter than expected " + row);
+ Preconditions.checkArgument(row.subSequence(0, prefix.length()).equals(prefix),
+ "Row does not have expected prefix " + row);
+ Preconditions.checkArgument(hasHash(row), "Row does not have expected hash " + row);
+ return row.subSequence(prefix.length() + 5, row.length());
+ }
+
+ private static String genHash(Bytes row) {
+ int hash = Hashing.murmur3_32().hashBytes(row.toArray()).asInt();
+ hash = hash & 0x7fffffff;
+ // base 36 gives a lot more bins in 4 bytes than hex, but it is still human readable which is
+ // nice for debugging.
+ String hashString =
+ Strings.padStart(Integer.toString(hash, Character.MAX_RADIX), HASH_LEN, '0');
+ hashString = hashString.substring(hashString.length() - HASH_LEN);
+
+ return hashString;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
index 0000000,ac04f80..dffa713
mode 000000,100644..100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
@@@ -1,0 -1,276 +1,277 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ package org.apache.fluo.recipes.core.export;
+
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.regex.Pattern;
+
+ import com.google.common.base.Preconditions;
+ import com.google.common.hash.Hashing;
+ import org.apache.fluo.api.client.TransactionBase;
+ import org.apache.fluo.api.config.FluoConfiguration;
+ import org.apache.fluo.api.config.ObserverConfiguration;
+ import org.apache.fluo.api.config.SimpleConfiguration;
+ import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ import org.apache.fluo.recipes.core.common.RowRange;
+ import org.apache.fluo.recipes.core.common.TransientRegistry;
+ import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+ /**
+ * @since 1.0.0
+ */
+ public class ExportQueue<K, V> {
+
+ private static final String RANGE_BEGIN = "#";
+ private static final String RANGE_END = ":~";
+
+ private int numBuckets;
+ private SimpleSerializer serializer;
+ private String queueId;
+
+ // usage hint : could be created once in an observers init method
+ // usage hint : maybe have a queue for each type of data being exported???
+ // maybe less queues are
+ // more efficient though because more batching at export time??
+ ExportQueue(Options opts, SimpleSerializer serializer) throws Exception {
+ // TODO sanity check key type based on type params
+ // TODO defer creating classes until needed.. so that its not done during Fluo init
+ this.queueId = opts.queueId;
+ this.numBuckets = opts.numBuckets;
+ this.serializer = serializer;
+ }
+
+ public void add(TransactionBase tx, K key, V value) {
+ addAll(tx, Collections.singleton(new Export<>(key, value)).iterator());
+ }
+
+ public void addAll(TransactionBase tx, Iterator<Export<K, V>> exports) {
+
+ Set<Integer> bucketsNotified = new HashSet<>();
+ while (exports.hasNext()) {
+ Export<K, V> export = exports.next();
+
+ byte[] k = serializer.serialize(export.getKey());
+ byte[] v = serializer.serialize(export.getValue());
+
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ int bucketId = Math.abs(hash % numBuckets);
+
+ ExportBucket bucket = new ExportBucket(tx, queueId, bucketId, numBuckets);
+ bucket.add(tx.getStartTimestamp(), k, v);
+
+ if (!bucketsNotified.contains(bucketId)) {
+ bucket.notifyExportObserver();
+ bucketsNotified.add(bucketId);
+ }
+ }
+ }
+
+ public static <K2, V2> ExportQueue<K2, V2> getInstance(String exportQueueId,
+ SimpleConfiguration appConfig) {
+ Options opts = new Options(exportQueueId, appConfig);
+ try {
+ return new ExportQueue<>(opts, SimpleSerializer.getInstance(appConfig));
+ } catch (Exception e) {
+ // TODO
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Call this method before initializing Fluo.
+ *
+ * @param fluoConfig The configuration that will be used to initialize fluo.
+ */
+ public static void configure(FluoConfiguration fluoConfig, Options opts) {
+ SimpleConfiguration appConfig = fluoConfig.getAppConfiguration();
+ opts.save(appConfig);
+
+ fluoConfig.addObserver(new ObserverConfiguration(ExportObserver.class.getName())
+ .setParameters(Collections.singletonMap("queueId", opts.queueId)));
+
+ Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
+ Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
+
+ new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("exportQueue."
+ + opts.queueId, new RowRange(exportRangeStart, exportRangeStop));
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for all previously configured export queues.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
+
- public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
++ public static TableOptimizations getTableOptimizations(SimpleConfiguration appConfig) {
+ HashSet<String> queueIds = new HashSet<>();
+ appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
+ k -> queueIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
+
- Pirtos pirtos = new Pirtos();
- queueIds.forEach(qid -> pirtos.merge(getTableOptimizations(qid, appConfig)));
++ TableOptimizations tableOptim = new TableOptimizations();
++ queueIds.forEach(qid -> tableOptim.merge(getTableOptimizations(qid, appConfig)));
+
- return pirtos;
++ return tableOptim;
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for the specified export queue.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
- public static Pirtos getTableOptimizations(String queueId, SimpleConfiguration appConfig) {
++ public static TableOptimizations getTableOptimizations(String queueId,
++ SimpleConfiguration appConfig) {
+ Options opts = new Options(queueId, appConfig);
+
+ List<Bytes> splits = new ArrayList<>();
+
+ Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
+ Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
+
+ splits.add(exportRangeStart);
+ splits.add(exportRangeStop);
+
+ List<Bytes> exportSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ exportSplits.add(ExportBucket.generateBucketRow(opts.queueId, i, opts.numBuckets));
+ }
+ Collections.sort(exportSplits);
+ splits.addAll(exportSplits);
+
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
++ TableOptimizations tableOptim = new TableOptimizations();
++ tableOptim.setSplits(splits);
+
+ // the tablet with end row <queueId># does not contain any data for the export queue and
+ // should not be grouped with the export queue
- pirtos.setTabletGroupingRegex(Pattern.quote(queueId + ":"));
++ tableOptim.setTabletGroupingRegex(Pattern.quote(queueId + ":"));
+
- return pirtos;
++ return tableOptim;
+ }
+
+ public static class Options {
+
+ private static final String PREFIX = "recipes.exportQueue.";
+ static final long DEFAULT_BUFFER_SIZE = 1 << 20;
+ static final int DEFAULT_BUCKETS_PER_TABLET = 10;
+
+ int numBuckets;
+ Integer bucketsPerTablet = null;
+ Long bufferSize;
+
+ String keyType;
+ String valueType;
+ String exporterType;
+ String queueId;
+
+ Options(String queueId, SimpleConfiguration appConfig) {
+ this.queueId = queueId;
+
+ this.numBuckets = appConfig.getInt(PREFIX + queueId + ".buckets");
+ this.exporterType = appConfig.getString(PREFIX + queueId + ".exporter");
+ this.keyType = appConfig.getString(PREFIX + queueId + ".key");
+ this.valueType = appConfig.getString(PREFIX + queueId + ".val");
+ this.bufferSize = appConfig.getLong(PREFIX + queueId + ".bufferSize", DEFAULT_BUFFER_SIZE);
+ this.bucketsPerTablet =
+ appConfig.getInt(PREFIX + queueId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
+ }
+
+ public Options(String queueId, String exporterType, String keyType, String valueType,
+ int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+
+ this.queueId = queueId;
+ this.numBuckets = buckets;
+ this.exporterType = exporterType;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+
+ public <K, V> Options(String queueId, Class<? extends Exporter<K, V>> exporter,
+ Class<K> keyType, Class<V> valueType, int buckets) {
+ this(queueId, exporter.getName(), keyType.getName(), valueType.getName(), buckets);
+ }
+
+ /**
+ * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
+ * be used to actually deserialize and process the updates. This limit does not account for
+ * object overhead in java, which can be significant.
+ *
+ * <p>
+ * The way memory read is calculated is by summing the length of serialized key and value byte
+ * arrays. Once this sum exceeds the configured memory limit, no more export key values are
+ * processed in the current transaction. When not everything is processed, the observer
+ * processing exports will notify itself causing another transaction to continue processing
+ * later.
+ */
+ public Options setBufferSize(long bufferSize) {
+ Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
+ this.bufferSize = bufferSize;
+ return this;
+ }
+
+ long getBufferSize() {
+ if (bufferSize == null) {
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ return bufferSize;
+ }
+
+ /**
+ * Sets the number of buckets per tablet to generate. This affects how many split points will be
+ * generated when optimizing the Accumulo table.
+ *
+ */
+ public Options setBucketsPerTablet(int bucketsPerTablet) {
+ Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
+ + bucketsPerTablet);
+ this.bucketsPerTablet = bucketsPerTablet;
+ return this;
+ }
+
+ int getBucketsPerTablet() {
+ if (bucketsPerTablet == null) {
+ return DEFAULT_BUCKETS_PER_TABLET;
+ }
+
+ return bucketsPerTablet;
+ }
+
+ void save(SimpleConfiguration appConfig) {
+ appConfig.setProperty(PREFIX + queueId + ".buckets", numBuckets + "");
+ appConfig.setProperty(PREFIX + queueId + ".exporter", exporterType + "");
+ appConfig.setProperty(PREFIX + queueId + ".key", keyType);
+ appConfig.setProperty(PREFIX + queueId + ".val", valueType);
+
+ if (bufferSize != null) {
+ appConfig.setProperty(PREFIX + queueId + ".bufferSize", bufferSize);
+ }
+ if (bucketsPerTablet != null) {
+ appConfig.setProperty(PREFIX + queueId + ".bucketsPerTablet", bucketsPerTablet);
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
index 0000000,6cbc658..2fe4a7c
mode 000000,100644..100644
--- a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
@@@ -1,0 -1,657 +1,657 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ package org.apache.fluo.recipes.core.map;
+
+ import java.io.Serializable;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Map.Entry;
+ import java.util.Optional;
+ import java.util.Set;
+ import java.util.regex.Pattern;
+
+ import com.google.common.base.Preconditions;
+ import com.google.common.collect.ImmutableMap;
+ import com.google.common.collect.Iterators;
+ import com.google.common.collect.Sets;
+ import com.google.common.hash.Hashing;
+ import org.apache.fluo.api.client.SnapshotBase;
+ import org.apache.fluo.api.client.TransactionBase;
+ import org.apache.fluo.api.config.FluoConfiguration;
+ import org.apache.fluo.api.config.ObserverConfiguration;
+ import org.apache.fluo.api.config.ScannerConfiguration;
+ import org.apache.fluo.api.config.SimpleConfiguration;
+ import org.apache.fluo.api.data.Bytes;
+ import org.apache.fluo.api.data.BytesBuilder;
+ import org.apache.fluo.api.data.Column;
+ import org.apache.fluo.api.data.RowColumn;
+ import org.apache.fluo.api.data.RowColumnValue;
+ import org.apache.fluo.api.data.Span;
+ import org.apache.fluo.api.iterator.ColumnIterator;
+ import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ import org.apache.fluo.recipes.core.common.RowRange;
+ import org.apache.fluo.recipes.core.common.TransientRegistry;
+ import org.apache.fluo.recipes.core.impl.BucketUtil;
+ import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+ /**
+ * See the project level documentation for information about this recipe.
+ *
+ * @since 1.0.0
+ */
+ public class CollisionFreeMap<K, V> {
+
+ private static final String UPDATE_RANGE_END = ":u:~";
+
+ private static final String DATA_RANGE_END = ":d:~";
+
+ private String mapId;
+
+ private Class<K> keyType;
+ private Class<V> valType;
+ private SimpleSerializer serializer;
+ private Combiner<K, V> combiner;
+ UpdateObserver<K, V> updateObserver;
+ private long bufferSize;
+
+ static final Column UPDATE_COL = new Column("u", "v");
+ static final Column NEXT_COL = new Column("u", "next");
+
+ private int numBuckets = -1;
+
+ @SuppressWarnings("unchecked")
+ CollisionFreeMap(Options opts, SimpleSerializer serializer) throws Exception {
+
+ this.mapId = opts.mapId;
+ // TODO defer loading classes
+ // TODO centralize class loading
+ // TODO try to check type params
+ this.numBuckets = opts.numBuckets;
+ this.keyType = (Class<K>) getClass().getClassLoader().loadClass(opts.keyType);
+ this.valType = (Class<V>) getClass().getClassLoader().loadClass(opts.valueType);
+ this.combiner =
+ (Combiner<K, V>) getClass().getClassLoader().loadClass(opts.combinerType).newInstance();
+ this.serializer = serializer;
+ if (opts.updateObserverType != null) {
+ this.updateObserver =
+ getClass().getClassLoader().loadClass(opts.updateObserverType)
+ .asSubclass(UpdateObserver.class).newInstance();
+ } else {
+ this.updateObserver = new NullUpdateObserver<>();
+ }
+ this.bufferSize = opts.getBufferSize();
+ }
+
+ private V deserVal(Bytes val) {
+ return serializer.deserialize(val.toArray(), valType);
+ }
+
+ private Bytes getKeyFromUpdateRow(Bytes prefix, Bytes row) {
+ return row.subSequence(prefix.length(), row.length() - 8);
+ }
+
+ void process(TransactionBase tx, Bytes ntfyRow, Column col) throws Exception {
+
+ Bytes nextKey = tx.get(ntfyRow, NEXT_COL);
+
+ ScannerConfiguration sc = new ScannerConfiguration();
+
+ if (nextKey != null) {
+ Bytes startRow =
+ Bytes.newBuilder(ntfyRow.length() + nextKey.length()).append(ntfyRow).append(nextKey)
+ .toBytes();
+ Span tmpSpan = Span.prefix(ntfyRow);
+ Span nextSpan =
+ new Span(new RowColumn(startRow, UPDATE_COL), false, tmpSpan.getEnd(),
+ tmpSpan.isEndInclusive());
+ sc.setSpan(nextSpan);
+ } else {
+ sc.setSpan(Span.prefix(ntfyRow));
+ }
+
+ sc.setSpan(Span.prefix(ntfyRow));
+ sc.fetchColumn(UPDATE_COL.getFamily(), UPDATE_COL.getQualifier());
+ RowIterator iter = tx.get(sc);
+
+ Map<Bytes, List<Bytes>> updates = new HashMap<>();
+
+ long approxMemUsed = 0;
+
+ Bytes partiallyReadKey = null;
+
+ if (iter.hasNext()) {
+ Bytes lastKey = null;
+ while (iter.hasNext() && approxMemUsed < bufferSize) {
+ Entry<Bytes, ColumnIterator> rowCol = iter.next();
+ Bytes curRow = rowCol.getKey();
+
+ tx.delete(curRow, UPDATE_COL);
+
+ Bytes serializedKey = getKeyFromUpdateRow(ntfyRow, curRow);
+ lastKey = serializedKey;
+
+ List<Bytes> updateList = updates.get(serializedKey);
+ if (updateList == null) {
+ updateList = new ArrayList<>();
+ updates.put(serializedKey, updateList);
+ }
+
+ Bytes val = rowCol.getValue().next().getValue();
+ updateList.add(val);
+
+ approxMemUsed += curRow.length();
+ approxMemUsed += val.length();
+ }
+
+ if (iter.hasNext()) {
+ Entry<Bytes, ColumnIterator> rowCol = iter.next();
+ Bytes curRow = rowCol.getKey();
+
+ // check if more updates for last key
+ if (getKeyFromUpdateRow(ntfyRow, curRow).equals(lastKey)) {
+ // there are still more updates for this key
+ partiallyReadKey = lastKey;
+
+ // start next time at the current key
+ tx.set(ntfyRow, NEXT_COL, partiallyReadKey);
+ } else {
+ // start next time at the next possible key
+ Bytes nextPossible =
+ Bytes.newBuilder(lastKey.length() + 1).append(lastKey).append(new byte[] {0})
+ .toBytes();
+ tx.set(ntfyRow, NEXT_COL, nextPossible);
+ }
+
+ // may not read all data because of mem limit, so notify self
+ tx.setWeakNotification(ntfyRow, col);
+ } else if (nextKey != null) {
+ // clear nextKey
+ tx.delete(ntfyRow, NEXT_COL);
+ }
+ } else if (nextKey != null) {
+ tx.delete(ntfyRow, NEXT_COL);
+ }
+
+ byte[] dataPrefix = ntfyRow.toArray();
+ // TODO this is awful... no sanity check... hard to read
+ dataPrefix[Bytes.of(mapId).length() + 1] = 'd';
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(dataPrefix);
+ int rowPrefixLen = rowBuilder.getLength();
+
+ Set<Bytes> keysToFetch = updates.keySet();
+ if (partiallyReadKey != null) {
+ final Bytes prk = partiallyReadKey;
+ keysToFetch = Sets.filter(keysToFetch, b -> !b.equals(prk));
+ }
+ Map<Bytes, Map<Column, Bytes>> currentVals = getCurrentValues(tx, rowBuilder, keysToFetch);
+
+ ArrayList<Update<K, V>> updatesToReport = new ArrayList<>(updates.size());
+
+ for (Entry<Bytes, List<Bytes>> entry : updates.entrySet()) {
+ rowBuilder.setLength(rowPrefixLen);
+ Bytes currentValueRow = rowBuilder.append(entry.getKey()).toBytes();
+ Bytes currVal =
+ currentVals.getOrDefault(currentValueRow, Collections.emptyMap()).get(DATA_COLUMN);
+
+ Iterator<V> ui = Iterators.transform(entry.getValue().iterator(), this::deserVal);
+
+ K kd = serializer.deserialize(entry.getKey().toArray(), keyType);
+
+ if (partiallyReadKey != null && partiallyReadKey.equals(entry.getKey())) {
+ // not all updates were read for this key, so requeue the combined updates as an update
+ Optional<V> nv = combiner.combine(kd, ui);
+ if (nv.isPresent()) {
+ update(tx, Collections.singletonMap(kd, nv.get()));
+ }
+ } else {
+ Optional<V> nv = combiner.combine(kd, concat(ui, currVal));
+ Bytes newVal = nv.isPresent() ? Bytes.of(serializer.serialize(nv.get())) : null;
+ if (newVal != null ^ currVal != null || (currVal != null && !currVal.equals(newVal))) {
+ if (newVal == null) {
+ tx.delete(currentValueRow, DATA_COLUMN);
+ } else {
+ tx.set(currentValueRow, DATA_COLUMN, newVal);
+ }
+
+ Optional<V> cvd = Optional.ofNullable(currVal).map(this::deserVal);
+ updatesToReport.add(new Update<>(kd, cvd, nv));
+ }
+ }
+ }
+
+ // TODO could clear these as converted to objects to avoid double memory usage
+ updates.clear();
+ currentVals.clear();
+
+ if (updatesToReport.size() > 0) {
+ updateObserver.updatingValues(tx, updatesToReport.iterator());
+ }
+ }
+
+ private static final Column DATA_COLUMN = new Column("data", "current");
+
+ private Map<Bytes, Map<Column, Bytes>> getCurrentValues(TransactionBase tx, BytesBuilder prefix,
+ Set<Bytes> keySet) {
+
+ Set<Bytes> rows = new HashSet<>();
+
+ int prefixLen = prefix.getLength();
+ for (Bytes key : keySet) {
+ prefix.setLength(prefixLen);
+ rows.add(prefix.append(key).toBytes());
+ }
+
+ try {
+ return tx.get(rows, Collections.singleton(DATA_COLUMN));
+ } catch (IllegalArgumentException e) {
+ System.out.println(rows.size());
+ throw e;
+ }
+ }
+
+ private Iterator<V> concat(Iterator<V> updates, Bytes currentVal) {
+ if (currentVal == null) {
+ return updates;
+ }
+
+ return Iterators.concat(updates, Iterators.singletonIterator(deserVal(currentVal)));
+ }
+
+ /**
+ * This method will retrieve the current value for key and any outstanding updates and combine
+ * them using the configured {@link Combiner}. The result from the combiner is returned.
+ */
+ public V get(SnapshotBase tx, K key) {
+
+ byte[] k = serializer.serialize(key);
+
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId).append(":u:").append(bucketId).append(":").append(k);
+
+ ScannerConfiguration sc = new ScannerConfiguration();
+ sc.setSpan(Span.prefix(rowBuilder.toBytes()));
+
+ RowIterator iter = tx.get(sc);
+
+ Iterator<V> ui;
+
+ if (iter.hasNext()) {
+ ui = Iterators.transform(iter, e -> deserVal(e.getValue().next().getValue()));
+ } else {
+ ui = Collections.<V>emptyList().iterator();
+ }
+
+ rowBuilder.setLength(mapId.length());
+ rowBuilder.append(":d:").append(bucketId).append(":").append(k);
+
+ Bytes dataRow = rowBuilder.toBytes();
+
+ Bytes cv = tx.get(dataRow, DATA_COLUMN);
+
+ if (!ui.hasNext()) {
+ if (cv == null) {
+ return null;
+ } else {
+ return deserVal(cv);
+ }
+ }
+
+ return combiner.combine(key, concat(ui, cv)).orElse(null);
+ }
+
+ String getId() {
+ return mapId;
+ }
+
+ /**
+ * Queues updates for a collision free map. These updates will be made by an Observer executing
+ * another transaction. This method will not collide with other transaction queuing updates for
+ * the same keys.
+ *
+ * @param tx This transaction will be used to make the updates.
+ * @param updates The keys in the map should correspond to keys in the collision free map being
+ * updated. The values in the map will be queued for updating.
+ */
+ public void update(TransactionBase tx, Map<K, V> updates) {
+ Preconditions.checkState(numBuckets > 0, "Not initialized");
+
+ Set<String> buckets = new HashSet<>();
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId).append(":u:");
+ int prefixLength = rowBuilder.getLength();
+
+ byte[] startTs = encSeq(tx.getStartTimestamp());
+
+ for (Entry<K, V> entry : updates.entrySet()) {
+ byte[] k = serializer.serialize(entry.getKey());
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+ // reset to the common row prefix
+ rowBuilder.setLength(prefixLength);
+
+ Bytes row = rowBuilder.append(bucketId).append(":").append(k).append(startTs).toBytes();
+ Bytes val = Bytes.of(serializer.serialize(entry.getValue()));
+
+ // TODO set if not exists would be comforting here.... but
+ // collisions on bucketId+key+uuid should never occur
+ tx.set(row, UPDATE_COL, val);
+
+ buckets.add(bucketId);
+ }
+
+ for (String bucketId : buckets) {
+ rowBuilder.setLength(prefixLength);
+ rowBuilder.append(bucketId).append(":");
+
+ Bytes row = rowBuilder.toBytes();
+
+ tx.setWeakNotification(row, new Column("fluoRecipes", "cfm:" + mapId));
+ }
+ }
+
+ public static <K2, V2> CollisionFreeMap<K2, V2> getInstance(String mapId,
+ SimpleConfiguration appConf) {
+ Options opts = new Options(mapId, appConf);
+ try {
+ return new CollisionFreeMap<>(opts, SimpleSerializer.getInstance(appConf));
+ } catch (Exception e) {
+ // TODO
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * A @link {@link CollisionFreeMap} stores data in its own data format in the Fluo table. When
+ * initializing a Fluo table with something like Map Reduce or Spark, data will need to be written
+ * in this format. Thats the purpose of this method, it provide a simple class that can do this
+ * conversion.
+ *
+ */
+ public static <K2, V2> Initializer<K2, V2> getInitializer(String mapId, int numBuckets,
+ SimpleSerializer serializer) {
+ return new Initializer<>(mapId, numBuckets, serializer);
+ }
+
+ /**
+ * @see CollisionFreeMap#getInitializer(String, int, SimpleSerializer)
+ */
+ public static class Initializer<K2, V2> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String mapId;
+
+ private SimpleSerializer serializer;
+
+ private int numBuckets = -1;
+
+ private Initializer(String mapId, int numBuckets, SimpleSerializer serializer) {
+ this.mapId = mapId;
+ this.numBuckets = numBuckets;
+ this.serializer = serializer;
+ }
+
+ public RowColumnValue convert(K2 key, V2 val) {
+ byte[] k = serializer.serialize(key);
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+ BytesBuilder bb = Bytes.newBuilder();
+ Bytes row = bb.append(mapId).append(":d:").append(bucketId).append(":").append(k).toBytes();
+ byte[] v = serializer.serialize(val);
+
+ return new RowColumnValue(row, DATA_COLUMN, Bytes.of(v));
+ }
+ }
+
+ public static class Options {
+
+ static final long DEFAULT_BUFFER_SIZE = 1 << 22;
+ static final int DEFAULT_BUCKETS_PER_TABLET = 10;
+
+ int numBuckets;
+ Integer bucketsPerTablet = null;
+
+ Long bufferSize;
+
+ String keyType;
+ String valueType;
+ String combinerType;
+ String updateObserverType;
+ String mapId;
+
+ private static final String PREFIX = "recipes.cfm.";
+
+ Options(String mapId, SimpleConfiguration appConfig) {
+ this.mapId = mapId;
+
+ this.numBuckets = appConfig.getInt(PREFIX + mapId + ".buckets");
+ this.combinerType = appConfig.getString(PREFIX + mapId + ".combiner");
+ this.keyType = appConfig.getString(PREFIX + mapId + ".key");
+ this.valueType = appConfig.getString(PREFIX + mapId + ".val");
+ this.updateObserverType = appConfig.getString(PREFIX + mapId + ".updateObserver", null);
+ this.bufferSize = appConfig.getLong(PREFIX + mapId + ".bufferSize", DEFAULT_BUFFER_SIZE);
+ this.bucketsPerTablet =
+ appConfig.getInt(PREFIX + mapId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
+ }
+
+ public Options(String mapId, String combinerType, String keyType, String valType, int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+ Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
+
+ this.mapId = mapId;
+ this.numBuckets = buckets;
+ this.combinerType = combinerType;
+ this.updateObserverType = null;
+ this.keyType = keyType;
+ this.valueType = valType;
+ }
+
+ public Options(String mapId, String combinerType, String updateObserverType, String keyType,
+ String valueType, int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+ Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
+
+ this.mapId = mapId;
+ this.numBuckets = buckets;
+ this.combinerType = combinerType;
+ this.updateObserverType = updateObserverType;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ /**
+ * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
+ * be used to actually deserialize and process the updates. This limit does not account for
+ * object overhead in java, which can be significant.
+ *
+ * <p>
+ * The way memory read is calculated is by summing the length of serialized key and value byte
+ * arrays. Once this sum exceeds the configured memory limit, no more update key values are
+ * processed in the current transaction. When not everything is processed, the observer
+ * processing updates will notify itself causing another transaction to continue processing
+ * later
+ */
+ public Options setBufferSize(long bufferSize) {
+ Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
+ this.bufferSize = bufferSize;
+ return this;
+ }
+
+ long getBufferSize() {
+ if (bufferSize == null) {
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ return bufferSize;
+ }
+
+ /**
+ * Sets the number of buckets per tablet to generate. This affects how many split points will be
+ * generated when optimizing the Accumulo table.
+ *
+ */
+ public Options setBucketsPerTablet(int bucketsPerTablet) {
+ Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
+ + bucketsPerTablet);
+ this.bucketsPerTablet = bucketsPerTablet;
+ return this;
+ }
+
+ int getBucketsPerTablet() {
+ if (bucketsPerTablet == null) {
+ return DEFAULT_BUCKETS_PER_TABLET;
+ }
+
+ return bucketsPerTablet;
+ }
+
+ public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner, Class<K> keyType,
+ Class<V> valueType, int buckets) {
+ this(mapId, combiner.getName(), keyType.getName(), valueType.getName(), buckets);
+ }
+
+ public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner,
+ Class<? extends UpdateObserver<K, V>> updateObserver, Class<K> keyType, Class<V> valueType,
+ int buckets) {
+ this(mapId, combiner.getName(), updateObserver.getName(), keyType.getName(), valueType
+ .getName(), buckets);
+ }
+
+ void save(SimpleConfiguration appConfig) {
+ appConfig.setProperty(PREFIX + mapId + ".buckets", numBuckets + "");
+ appConfig.setProperty(PREFIX + mapId + ".combiner", combinerType + "");
+ appConfig.setProperty(PREFIX + mapId + ".key", keyType);
+ appConfig.setProperty(PREFIX + mapId + ".val", valueType);
+ if (updateObserverType != null) {
+ appConfig.setProperty(PREFIX + mapId + ".updateObserver", updateObserverType + "");
+ }
+ if (bufferSize != null) {
+ appConfig.setProperty(PREFIX + mapId + ".bufferSize", bufferSize);
+ }
+ if (bucketsPerTablet != null) {
+ appConfig.setProperty(PREFIX + mapId + ".bucketsPerTablet", bucketsPerTablet);
+ }
+ }
+ }
+
+ /**
+ * This method configures a collision free map for use. It must be called before initializing
+ * Fluo.
+ */
+ public static void configure(FluoConfiguration fluoConfig, Options opts) {
+ opts.save(fluoConfig.getAppConfiguration());
+ fluoConfig.addObserver(new ObserverConfiguration(CollisionFreeMapObserver.class.getName())
+ .setParameters(ImmutableMap.of("mapId", opts.mapId)));
+
+ Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
+ Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
+
+ new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("cfm." + opts.mapId,
+ new RowRange(dataRangeEnd, updateRangeEnd));
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for all previously configured collision free maps.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
- public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
++ public static TableOptimizations getTableOptimizations(SimpleConfiguration appConfig) {
+ HashSet<String> mapIds = new HashSet<>();
+ appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
+ k -> mapIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
+
- Pirtos pirtos = new Pirtos();
- mapIds.forEach(mid -> pirtos.merge(getTableOptimizations(mid, appConfig)));
++ TableOptimizations tableOptim = new TableOptimizations();
++ mapIds.forEach(mid -> tableOptim.merge(getTableOptimizations(mid, appConfig)));
+
- return pirtos;
++ return tableOptim;
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for the specified collisiong free map.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
- public static Pirtos getTableOptimizations(String mapId, SimpleConfiguration appConfig) {
++ public static TableOptimizations getTableOptimizations(String mapId, SimpleConfiguration appConfig) {
+ Options opts = new Options(mapId, appConfig);
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId);
+
+ List<Bytes> dataSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
+ rowBuilder.setLength(mapId.length());
+ dataSplits.add(rowBuilder.append(":d:").append(bucketId).toBytes());
+ }
+ Collections.sort(dataSplits);
+
+ List<Bytes> updateSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
+ rowBuilder.setLength(mapId.length());
+ updateSplits.add(rowBuilder.append(":u:").append(bucketId).toBytes());
+ }
+ Collections.sort(updateSplits);
+
+ Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
+ Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
+
+ List<Bytes> splits = new ArrayList<>();
+ splits.add(dataRangeEnd);
+ splits.add(updateRangeEnd);
+ splits.addAll(dataSplits);
+ splits.addAll(updateSplits);
+
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
++ TableOptimizations tableOptim = new TableOptimizations();
++ tableOptim.setSplits(splits);
+
- pirtos.setTabletGroupingRegex(Pattern.quote(mapId + ":") + "[du]:");
++ tableOptim.setTabletGroupingRegex(Pattern.quote(mapId + ":") + "[du]:");
+
- return pirtos;
++ return tableOptim;
+ }
+
+ private static byte[] encSeq(long l) {
+ byte[] ret = new byte[8];
+ ret[0] = (byte) (l >>> 56);
+ ret[1] = (byte) (l >>> 48);
+ ret[2] = (byte) (l >>> 40);
+ ret[3] = (byte) (l >>> 32);
+ ret[4] = (byte) (l >>> 24);
+ ret[5] = (byte) (l >>> 16);
+ ret[6] = (byte) (l >>> 8);
+ ret[7] = (byte) (l >>> 0);
+ return ret;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
index 0000000,5a1f5fe..8ceda12
mode 000000,100644..100644
--- a/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/common/TestGrouping.java
@@@ -1,0 -1,92 +1,93 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ package org.apache.fluo.recipes.core.common;
+
+ import java.util.Set;
+ import java.util.regex.Matcher;
+ import java.util.regex.Pattern;
+
+ import com.google.common.collect.ImmutableSet;
+ import org.apache.fluo.api.config.FluoConfiguration;
+ import org.apache.fluo.api.data.Bytes;
+ import org.apache.fluo.recipes.core.export.ExportQueue;
+ import org.apache.fluo.recipes.core.map.CollisionFreeMap;
+ import org.apache.fluo.recipes.core.map.CollisionFreeMap.Options;
+ import org.junit.Assert;
+ import org.junit.Test;
+
+ public class TestGrouping {
+ @Test
+ public void testTabletGrouping() {
+ FluoConfiguration conf = new FluoConfiguration();
+
+ CollisionFreeMap.configure(conf, new Options("m1", "ct", "kt", "vt", 119));
+ CollisionFreeMap.configure(conf, new Options("m2", "ct", "kt", "vt", 3));
+
+ ExportQueue.configure(conf, new ExportQueue.Options("eq1", "et", "kt", "vt", 7));
+ ExportQueue.configure(conf, new ExportQueue.Options("eq2", "et", "kt", "vt", 3));
+
- Pirtos pirtos = CollisionFreeMap.getTableOptimizations(conf.getAppConfiguration());
- pirtos.merge(ExportQueue.getTableOptimizations(conf.getAppConfiguration()));
++ TableOptimizations tableOptim =
++ CollisionFreeMap.getTableOptimizations(conf.getAppConfiguration());
++ tableOptim.merge(ExportQueue.getTableOptimizations(conf.getAppConfiguration()));
+
- Pattern pattern = Pattern.compile(pirtos.getTabletGroupingRegex());
++ Pattern pattern = Pattern.compile(tableOptim.getTabletGroupingRegex());
+
+ Assert.assertEquals("m1:u:", group(pattern, "m1:u:f0c"));
+ Assert.assertEquals("m1:d:", group(pattern, "m1:d:f0c"));
+ Assert.assertEquals("m2:u:", group(pattern, "m2:u:abc"));
+ Assert.assertEquals("m2:d:", group(pattern, "m2:d:590"));
+ Assert.assertEquals("none", group(pattern, "m3:d:590"));
+
+ Assert.assertEquals("eq1:", group(pattern, "eq1:f0c"));
+ Assert.assertEquals("eq2:", group(pattern, "eq2:f0c"));
+ Assert.assertEquals("none", group(pattern, "eq3:f0c"));
+
+ // validate the assumptions this test is making
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1#")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2#")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:u:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:d:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:u:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:d:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("eq1#")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("eq2#")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("eq1:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("eq2:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("m1:u:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("m1:d:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("m2:u:~")));
++ Assert.assertTrue(tableOptim.getSplits().contains(Bytes.of("m2:d:~")));
+
+ Set<String> expectedGroups =
+ ImmutableSet.of("m1:u:", "m1:d:", "m2:u:", "m2:d:", "eq1:", "eq2:");
+
+ // ensure all splits group as expected
- for (Bytes split : pirtos.getSplits()) {
++ for (Bytes split : tableOptim.getSplits()) {
+ String g = group(pattern, split.toString());
+
+ if (expectedGroups.contains(g)) {
+ Assert.assertTrue(split.toString().startsWith(g));
+ } else {
+ Assert.assertEquals("none", g);
+ Assert.assertTrue(split.toString().equals("eq1#") || split.toString().equals("eq2#"));
+ }
+
+ }
+
+ }
+
+ private String group(Pattern pattern, String endRow) {
+ Matcher m = pattern.matcher(endRow);
+ if (m.matches() && m.groupCount() == 1) {
+ return m.group(1);
+ }
+ return "none";
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
index 0000000,a359598..8259469
mode 000000,100644..100644
--- a/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
+++ b/modules/core/src/test/java/org/apache/fluo/recipes/core/map/SplitsTest.java
@@@ -1,0 -1,75 +1,76 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ package org.apache.fluo.recipes.core.map;
+
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
+ import java.util.List;
+
+ import com.google.common.collect.Lists;
+ import org.apache.fluo.api.config.FluoConfiguration;
+ import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
+ import org.apache.fluo.recipes.core.map.CollisionFreeMap.Options;
+ import org.junit.Assert;
+ import org.junit.Test;
+
+ public class SplitsTest {
+ private static List<Bytes> sort(List<Bytes> in) {
+ ArrayList<Bytes> out = new ArrayList<>(in);
+ Collections.sort(out);
+ return out;
+ }
+
+ @Test
+ public void testSplits() {
+
+ Options opts = new Options("foo", WordCountCombiner.class, String.class, Long.class, 3);
+ opts.setBucketsPerTablet(1);
+ FluoConfiguration fluoConfig = new FluoConfiguration();
+ CollisionFreeMap.configure(fluoConfig, opts);
+
- Pirtos pirtos1 =
++ TableOptimizations tableOptim1 =
+ CollisionFreeMap.getTableOptimizations("foo", fluoConfig.getAppConfiguration());
+ List<Bytes> expected1 =
+ Lists.transform(
+ Arrays.asList("foo:d:1", "foo:d:2", "foo:d:~", "foo:u:1", "foo:u:2", "foo:u:~"),
+ Bytes::of);
+
- Assert.assertEquals(expected1, sort(pirtos1.getSplits()));
++ Assert.assertEquals(expected1, sort(tableOptim1.getSplits()));
+
+ Options opts2 = new Options("bar", WordCountCombiner.class, String.class, Long.class, 6);
+ opts2.setBucketsPerTablet(2);
+ CollisionFreeMap.configure(fluoConfig, opts2);
+
- Pirtos pirtos2 =
++ TableOptimizations tableOptim2 =
+ CollisionFreeMap.getTableOptimizations("bar", fluoConfig.getAppConfiguration());
+ List<Bytes> expected2 =
+ Lists.transform(
+ Arrays.asList("bar:d:2", "bar:d:4", "bar:d:~", "bar:u:2", "bar:u:4", "bar:u:~"),
+ Bytes::of);
- Assert.assertEquals(expected2, sort(pirtos2.getSplits()));
++ Assert.assertEquals(expected2, sort(tableOptim2.getSplits()));
+
- Pirtos pirtos3 = CollisionFreeMap.getTableOptimizations(fluoConfig.getAppConfiguration());
++ TableOptimizations tableOptim3 =
++ CollisionFreeMap.getTableOptimizations(fluoConfig.getAppConfiguration());
+
+ ArrayList<Bytes> expected3 = new ArrayList<>(expected2);
+ expected3.addAll(expected1);
+
- Assert.assertEquals(expected3, sort(pirtos3.getSplits()));
++ Assert.assertEquals(expected3, sort(tableOptim3.getSplits()));
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/22354d0f/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
----------------------------------------------------------------------
diff --cc modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
index 07cff67,c1adc3b..00795f4
--- a/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
+++ b/modules/test/src/main/java/org/apache/fluo/recipes/test/AccumuloExportITBase.java
@@@ -31,7 -31,7 +31,7 @@@ import org.apache.fluo.api.client.FluoF
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.mini.MiniFluo;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
- import org.apache.fluo.recipes.common.TableOptimizations;
-import org.apache.fluo.recipes.core.common.Pirtos;
++import org.apache.fluo.recipes.core.common.TableOptimizations;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
[07/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
Updated package names in core module
* Packages now have 'org.apache.recipes.core.' prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/commit/beea3f96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/tree/beea3f96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/diff/beea3f96
Branch: refs/heads/master
Commit: beea3f96e5f633341d977d327f6aa1a0f12307f3
Parents: f1dce14
Author: Mike Walch <mw...@gmail.com>
Authored: Fri Jul 15 11:31:28 2016 -0400
Committer: Mike Walch <mw...@gmail.com>
Committed: Fri Jul 15 11:46:22 2016 -0400
----------------------------------------------------------------------
docs/export-queue.md | 2 +-
docs/row-hasher.md | 4 +-
docs/serialization.md | 2 +-
docs/transient.md | 2 +-
.../recipes/accumulo/cmds/CompactTransient.java | 4 +-
.../recipes/accumulo/cmds/OptimizeTable.java | 2 +-
.../accumulo/export/AccumuloExporter.java | 4 +-
.../accumulo/export/ReplicationExport.java | 4 +-
.../recipes/accumulo/ops/TableOperations.java | 6 +-
.../org/apache/fluo/recipes/common/Pirtos.java | 83 ---
.../apache/fluo/recipes/common/RowRange.java | 82 ---
.../fluo/recipes/common/TransientRegistry.java | 79 ---
.../apache/fluo/recipes/core/common/Pirtos.java | 83 +++
.../fluo/recipes/core/common/RowRange.java | 82 +++
.../recipes/core/common/TransientRegistry.java | 79 +++
.../fluo/recipes/core/data/RowHasher.java | 135 ++++
.../apache/fluo/recipes/core/export/Export.java | 38 ++
.../fluo/recipes/core/export/ExportBucket.java | 203 ++++++
.../fluo/recipes/core/export/ExportEntry.java | 22 +
.../recipes/core/export/ExportObserver.java | 140 ++++
.../fluo/recipes/core/export/ExportQueue.java | 273 ++++++++
.../fluo/recipes/core/export/Exporter.java | 64 ++
.../recipes/core/export/SequencedExport.java | 29 +
.../fluo/recipes/core/impl/BucketUtil.java | 24 +
.../fluo/recipes/core/map/CollisionFreeMap.java | 657 +++++++++++++++++++
.../core/map/CollisionFreeMapObserver.java | 53 ++
.../apache/fluo/recipes/core/map/Combiner.java | 31 +
.../recipes/core/map/NullUpdateObserver.java | 25 +
.../apache/fluo/recipes/core/map/Update.java | 43 ++
.../fluo/recipes/core/map/UpdateObserver.java | 34 +
.../core/serialization/SimpleSerializer.java | 56 ++
.../fluo/recipes/core/transaction/LogEntry.java | 114 ++++
.../core/transaction/RecordingTransaction.java | 64 ++
.../transaction/RecordingTransactionBase.java | 250 +++++++
.../fluo/recipes/core/transaction/TxLog.java | 79 +++
.../apache/fluo/recipes/core/types/Encoder.java | 86 +++
.../fluo/recipes/core/types/StringEncoder.java | 86 +++
.../fluo/recipes/core/types/TypeLayer.java | 488 ++++++++++++++
.../fluo/recipes/core/types/TypedLoader.java | 45 ++
.../fluo/recipes/core/types/TypedObserver.java | 46 ++
.../fluo/recipes/core/types/TypedSnapshot.java | 38 ++
.../recipes/core/types/TypedSnapshotBase.java | 555 ++++++++++++++++
.../recipes/core/types/TypedTransaction.java | 46 ++
.../core/types/TypedTransactionBase.java | 278 ++++++++
.../org/apache/fluo/recipes/data/RowHasher.java | 135 ----
.../org/apache/fluo/recipes/export/Export.java | 38 --
.../fluo/recipes/export/ExportBucket.java | 203 ------
.../apache/fluo/recipes/export/ExportEntry.java | 22 -
.../fluo/recipes/export/ExportObserver.java | 140 ----
.../apache/fluo/recipes/export/ExportQueue.java | 273 --------
.../apache/fluo/recipes/export/Exporter.java | 64 --
.../fluo/recipes/export/SequencedExport.java | 29 -
.../apache/fluo/recipes/impl/BucketUtil.java | 24 -
.../fluo/recipes/map/CollisionFreeMap.java | 657 -------------------
.../recipes/map/CollisionFreeMapObserver.java | 53 --
.../org/apache/fluo/recipes/map/Combiner.java | 31 -
.../fluo/recipes/map/NullUpdateObserver.java | 25 -
.../org/apache/fluo/recipes/map/Update.java | 43 --
.../apache/fluo/recipes/map/UpdateObserver.java | 34 -
.../recipes/serialization/SimpleSerializer.java | 56 --
.../fluo/recipes/transaction/LogEntry.java | 114 ----
.../transaction/RecordingTransaction.java | 64 --
.../transaction/RecordingTransactionBase.java | 250 -------
.../apache/fluo/recipes/transaction/TxLog.java | 79 ---
.../org/apache/fluo/recipes/types/Encoder.java | 86 ---
.../fluo/recipes/types/StringEncoder.java | 86 ---
.../apache/fluo/recipes/types/TypeLayer.java | 488 --------------
.../apache/fluo/recipes/types/TypedLoader.java | 45 --
.../fluo/recipes/types/TypedObserver.java | 46 --
.../fluo/recipes/types/TypedSnapshot.java | 38 --
.../fluo/recipes/types/TypedSnapshotBase.java | 555 ----------------
.../fluo/recipes/types/TypedTransaction.java | 46 --
.../recipes/types/TypedTransactionBase.java | 278 --------
.../fluo/recipes/common/TestGrouping.java | 94 ---
.../recipes/common/TransientRegistryTest.java | 48 --
.../fluo/recipes/core/common/TestGrouping.java | 92 +++
.../core/common/TransientRegistryTest.java | 48 ++
.../fluo/recipes/core/data/RowHasherTest.java | 62 ++
.../recipes/core/export/DocumentLoader.java | 36 +
.../recipes/core/export/DocumentObserver.java | 63 ++
.../recipes/core/export/ExportBufferIT.java | 106 +++
.../fluo/recipes/core/export/ExportQueueIT.java | 114 ++++
.../recipes/core/export/ExportTestBase.java | 286 ++++++++
.../recipes/core/export/GsonSerializer.java | 42 ++
.../fluo/recipes/core/export/OptionsTest.java | 51 ++
.../fluo/recipes/core/export/RefInfo.java | 26 +
.../fluo/recipes/core/export/RefUpdates.java | 43 ++
.../fluo/recipes/core/map/BigUpdateIT.java | 214 ++++++
.../recipes/core/map/CollisionFreeMapIT.java | 361 ++++++++++
.../fluo/recipes/core/map/DocumentLoader.java | 35 +
.../fluo/recipes/core/map/DocumentObserver.java | 89 +++
.../fluo/recipes/core/map/OptionsTest.java | 51 ++
.../fluo/recipes/core/map/SplitsTest.java | 75 +++
.../fluo/recipes/core/map/TestSerializer.java | 45 ++
.../recipes/core/map/WordCountCombiner.java | 36 +
.../recipes/core/map/WordCountObserver.java | 47 ++
.../transaction/RecordingTransactionTest.java | 227 +++++++
.../fluo/recipes/core/types/MockSnapshot.java | 30 +
.../recipes/core/types/MockSnapshotBase.java | 202 ++++++
.../recipes/core/types/MockTransaction.java | 36 +
.../recipes/core/types/MockTransactionBase.java | 90 +++
.../fluo/recipes/core/types/TypeLayerTest.java | 494 ++++++++++++++
.../apache/fluo/recipes/data/RowHasherTest.java | 62 --
.../fluo/recipes/export/DocumentLoader.java | 36 -
.../fluo/recipes/export/DocumentObserver.java | 63 --
.../fluo/recipes/export/ExportBufferIT.java | 106 ---
.../fluo/recipes/export/ExportQueueIT.java | 114 ----
.../fluo/recipes/export/ExportTestBase.java | 286 --------
.../fluo/recipes/export/GsonSerializer.java | 42 --
.../apache/fluo/recipes/export/OptionsTest.java | 51 --
.../org/apache/fluo/recipes/export/RefInfo.java | 26 -
.../apache/fluo/recipes/export/RefUpdates.java | 43 --
.../apache/fluo/recipes/map/BigUpdateIT.java | 214 ------
.../fluo/recipes/map/CollisionFreeMapIT.java | 361 ----------
.../apache/fluo/recipes/map/DocumentLoader.java | 35 -
.../fluo/recipes/map/DocumentObserver.java | 89 ---
.../apache/fluo/recipes/map/OptionsTest.java | 51 --
.../org/apache/fluo/recipes/map/SplitsTest.java | 75 ---
.../apache/fluo/recipes/map/TestSerializer.java | 45 --
.../fluo/recipes/map/WordCountCombiner.java | 36 -
.../fluo/recipes/map/WordCountObserver.java | 47 --
.../transaction/RecordingTransactionTest.java | 227 -------
.../apache/fluo/recipes/types/MockSnapshot.java | 30 -
.../fluo/recipes/types/MockSnapshotBase.java | 202 ------
.../fluo/recipes/types/MockTransaction.java | 36 -
.../fluo/recipes/types/MockTransactionBase.java | 90 ---
.../fluo/recipes/types/TypeLayerTest.java | 494 --------------
.../recipes/kryo/KryoSimplerSerializer.java | 2 +-
.../serialization/KryoSimpleSerializerTest.java | 45 ++
.../serialization/KryoSimpleSerializerTest.java | 45 --
.../fluo/recipes/test/AccumuloExportITBase.java | 2 +-
.../recipes/test/export/AccumuloExporterIT.java | 2 +-
.../test/export/AccumuloReplicatorIT.java | 10 +-
133 files changed, 7315 insertions(+), 7317 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/docs/export-queue.md
----------------------------------------------------------------------
diff --git a/docs/export-queue.md b/docs/export-queue.md
index b85895f..0120113 100644
--- a/docs/export-queue.md
+++ b/docs/export-queue.md
@@ -279,7 +279,7 @@ example of write skew mentioned in the Percolater paper.
1. TH1 : tx1.set(`rowA`,`fam1:qual2`, val1)
1. TH2 : tx2.set(`rowB`,`fam1:qual2`, val2)
-[1]: ../modules/core/src/main/java/org/apache/fluo/recipes/export/Exporter.java
+[1]: ../modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
[2]: https://en.wikipedia.org/wiki/Serializability
[3]: accumulo-export.md
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/docs/row-hasher.md
----------------------------------------------------------------------
diff --git a/docs/row-hasher.md b/docs/row-hasher.md
index adb423f..8db8af6 100644
--- a/docs/row-hasher.md
+++ b/docs/row-hasher.md
@@ -31,8 +31,8 @@ balancing of the prefix.
```java
import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.common.Pirtos;
-import org.apache.fluo.recipes.data.RowHasher;
+import org.apache.fluo.recipes.core.common.Pirtos;
+import org.apache.fluo.recipes.core.data.RowHasher;
public class RowHasherExample {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/docs/serialization.md
----------------------------------------------------------------------
diff --git a/docs/serialization.md b/docs/serialization.md
index d7e3b42..9b664a0 100644
--- a/docs/serialization.md
+++ b/docs/serialization.md
@@ -69,5 +69,5 @@ how to do this.
```
[1]: https://github.com/EsotericSoftware/kryo
-[2]: ../modules/core/src/main/java/org/apache/fluo/recipes/serialization/SimpleSerializer.java
+[2]: ../modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
[3]: https://github.com/EsotericSoftware/kryo#registration
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/docs/transient.md
----------------------------------------------------------------------
diff --git a/docs/transient.md b/docs/transient.md
index be68493..d0ac845 100644
--- a/docs/transient.md
+++ b/docs/transient.md
@@ -76,5 +76,5 @@ first range takes 20 seconds to compact, then it will be compacted again in 600
seconds. If the second range takes 80 seconds to compact, then it will be
compacted again in 800 seconds.
-[1]: ../modules/core/src/main/java/org/apache/fluo/recipes/common/TransientRegistry.java
+[1]: ../modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
[2]: ../modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
index 7ce53c8..54e05bc 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/CompactTransient.java
@@ -27,8 +27,8 @@ import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
-import org.apache.fluo.recipes.common.RowRange;
-import org.apache.fluo.recipes.common.TransientRegistry;
+import org.apache.fluo.recipes.core.common.RowRange;
+import org.apache.fluo.recipes.core.common.TransientRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
index 6b5276b..bf17cfd 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/cmds/OptimizeTable.java
@@ -19,7 +19,7 @@ import javax.inject.Inject;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
-import org.apache.fluo.recipes.common.Pirtos;
+import org.apache.fluo.recipes.core.common.Pirtos;
public class OptimizeTable {
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
index cd112eb..1788403 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/AccumuloExporter.java
@@ -22,8 +22,8 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
import org.apache.fluo.api.observer.Observer.Context;
-import org.apache.fluo.recipes.export.Exporter;
-import org.apache.fluo.recipes.export.SequencedExport;
+import org.apache.fluo.recipes.core.export.Exporter;
+import org.apache.fluo.recipes.core.export.SequencedExport;
/**
* An {@link Exporter} that takes {@link AccumuloExport} objects and writes mutations to Accumulo
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
index ec2f4ac..9a3b196 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/ReplicationExport.java
@@ -25,8 +25,8 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
-import org.apache.fluo.recipes.transaction.LogEntry;
-import org.apache.fluo.recipes.transaction.TxLog;
+import org.apache.fluo.recipes.core.transaction.LogEntry;
+import org.apache.fluo.recipes.core.transaction.TxLog;
/**
* An implementation of {@link AccumuloExport} that replicates a Fluo table to Accumulo using a
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
index e521014..62abd7e 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/ops/TableOperations.java
@@ -28,9 +28,9 @@ import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.SimpleConfiguration;
import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.common.Pirtos;
-import org.apache.fluo.recipes.common.RowRange;
-import org.apache.fluo.recipes.common.TransientRegistry;
+import org.apache.fluo.recipes.core.common.Pirtos;
+import org.apache.fluo.recipes.core.common.RowRange;
+import org.apache.fluo.recipes.core.common.TransientRegistry;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/common/Pirtos.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/common/Pirtos.java b/modules/core/src/main/java/org/apache/fluo/recipes/common/Pirtos.java
deleted file mode 100644
index bdb9e60..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/common/Pirtos.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.common;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.FluoFactory;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.export.ExportQueue;
-import org.apache.fluo.recipes.map.CollisionFreeMap;
-
-/**
- * Post initialization recommended table optimizations.
- */
-
-public class Pirtos {
- private List<Bytes> splits = new ArrayList<>();
- private String tabletGroupingRegex = "";
-
- public void setSplits(List<Bytes> splits) {
- this.splits.clear();
- this.splits.addAll(splits);
- }
-
- /**
- * @return A recommended set of splits points to add to a Fluo table after initialization.
- */
- public List<Bytes> getSplits() {
- return Collections.unmodifiableList(splits);
- }
-
- public void setTabletGroupingRegex(String tgr) {
- Objects.requireNonNull(tgr);
- this.tabletGroupingRegex = tgr;
- }
-
- public String getTabletGroupingRegex() {
- return "(" + tabletGroupingRegex + ").*";
- }
-
- public void merge(Pirtos other) {
- splits.addAll(other.splits);
- if (tabletGroupingRegex.length() > 0 && other.tabletGroupingRegex.length() > 0) {
- tabletGroupingRegex += "|" + other.tabletGroupingRegex;
- } else {
- tabletGroupingRegex += other.tabletGroupingRegex;
- }
- }
-
- /**
- * A utility method to get table optimizations for all configured recipes.
- */
- public static Pirtos getConfiguredOptimizations(FluoConfiguration fluoConfig) {
- try (FluoClient client = FluoFactory.newClient(fluoConfig)) {
- SimpleConfiguration appConfig = client.getAppConfiguration();
- Pirtos pirtos = new Pirtos();
-
- pirtos.merge(ExportQueue.getTableOptimizations(appConfig));
- pirtos.merge(CollisionFreeMap.getTableOptimizations(appConfig));
-
- return pirtos;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/common/RowRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/common/RowRange.java b/modules/core/src/main/java/org/apache/fluo/recipes/common/RowRange.java
deleted file mode 100644
index b1fbc23..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/common/RowRange.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.common;
-
-import java.util.Objects;
-
-import org.apache.fluo.api.data.Bytes;
-
-public class RowRange {
- private final Bytes start;
- private final Bytes end;
-
- public RowRange(Bytes start, Bytes end) {
- Objects.requireNonNull(start);
- Objects.requireNonNull(end);
- this.start = start;
- this.end = end;
- }
-
- public Bytes getStart() {
- return start;
- }
-
- public Bytes getEnd() {
- return end;
- }
-
- @Override
- public int hashCode() {
- return start.hashCode() + 31 * end.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof RowRange) {
- RowRange or = (RowRange) o;
- return start.equals(or.start) && end.equals(or.end);
- }
-
- return false;
- }
-
- private static void encNonAscii(StringBuilder sb, Bytes bytes) {
- if (bytes == null) {
- sb.append("null");
- } else {
- for (int i = 0; i < bytes.length(); i++) {
- byte b = bytes.byteAt(i);
- if (b >= 32 && b <= 126 && b != '\\') {
- sb.append((char) b);
- } else {
- sb.append(String.format("\\x%02x", b & 0xff));
- }
- }
- }
- }
-
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("(");
- encNonAscii(sb, start);
- sb.append(", ");
- encNonAscii(sb, end);
- sb.append("]");
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/common/TransientRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/common/TransientRegistry.java b/modules/core/src/main/java/org/apache/fluo/recipes/common/TransientRegistry.java
deleted file mode 100644
index 681d783..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/common/TransientRegistry.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.common;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.xml.bind.DatatypeConverter;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.api.data.Bytes;
-
-/**
- * This class offers a standard way to register transient ranges. The project level documentation
- * provides a comprehensive overview.
- */
-
-public class TransientRegistry {
-
- private SimpleConfiguration appConfig;
- private static final String PREFIX = "recipes.transientRange.";
-
- /**
- * @param appConfig Fluo application config. Can be obtained from
- * {@link FluoConfiguration#getAppConfiguration()} before initializing fluo when adding
- * Transient ranges. After Fluo is initialized, app config can be obtained from
- * {@link FluoClient#getAppConfiguration()} or
- * {@link org.apache.fluo.api.observer.Observer.Context#getAppConfiguration()}
- */
- public TransientRegistry(SimpleConfiguration appConfig) {
- this.appConfig = appConfig;
- }
-
- /**
- * This method is expected to be called before Fluo is initialized to register transient ranges.
- *
- */
- public void addTransientRange(String id, RowRange range) {
- String start = DatatypeConverter.printHexBinary(range.getStart().toArray());
- String end = DatatypeConverter.printHexBinary(range.getEnd().toArray());
-
- appConfig.setProperty(PREFIX + id, start + ":" + end);
- }
-
- /**
- * This method is expected to be called after Fluo is initialized to get the ranges that were
- * registered before initialization.
- */
- public List<RowRange> getTransientRanges() {
- List<RowRange> ranges = new ArrayList<>();
- Iterator<String> keys = appConfig.getKeys(PREFIX.substring(0, PREFIX.length() - 1));
- while (keys.hasNext()) {
- String key = keys.next();
- String val = appConfig.getString(key);
- String[] sa = val.split(":");
- RowRange rowRange =
- new RowRange(Bytes.of(DatatypeConverter.parseHexBinary(sa[0])),
- Bytes.of(DatatypeConverter.parseHexBinary(sa[1])));
- ranges.add(rowRange);
- }
- return ranges;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
new file mode 100644
index 0000000..5488e29
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/Pirtos.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.common;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.FluoFactory;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.recipes.core.export.ExportQueue;
+import org.apache.fluo.recipes.core.map.CollisionFreeMap;
+
+/**
+ * Post initialization recommended table optimizations.
+ */
+
+public class Pirtos {
+ private List<Bytes> splits = new ArrayList<>();
+ private String tabletGroupingRegex = "";
+
+ public void setSplits(List<Bytes> splits) {
+ this.splits.clear();
+ this.splits.addAll(splits);
+ }
+
+ /**
+ * @return A recommended set of splits points to add to a Fluo table after initialization.
+ */
+ public List<Bytes> getSplits() {
+ return Collections.unmodifiableList(splits);
+ }
+
+ public void setTabletGroupingRegex(String tgr) {
+ Objects.requireNonNull(tgr);
+ this.tabletGroupingRegex = tgr;
+ }
+
+ public String getTabletGroupingRegex() {
+ return "(" + tabletGroupingRegex + ").*";
+ }
+
+ public void merge(Pirtos other) {
+ splits.addAll(other.splits);
+ if (tabletGroupingRegex.length() > 0 && other.tabletGroupingRegex.length() > 0) {
+ tabletGroupingRegex += "|" + other.tabletGroupingRegex;
+ } else {
+ tabletGroupingRegex += other.tabletGroupingRegex;
+ }
+ }
+
+ /**
+ * A utility method to get table optimizations for all configured recipes.
+ */
+ public static Pirtos getConfiguredOptimizations(FluoConfiguration fluoConfig) {
+ try (FluoClient client = FluoFactory.newClient(fluoConfig)) {
+ SimpleConfiguration appConfig = client.getAppConfiguration();
+ Pirtos pirtos = new Pirtos();
+
+ pirtos.merge(ExportQueue.getTableOptimizations(appConfig));
+ pirtos.merge(CollisionFreeMap.getTableOptimizations(appConfig));
+
+ return pirtos;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
new file mode 100644
index 0000000..913357b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/RowRange.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.common;
+
+import java.util.Objects;
+
+import org.apache.fluo.api.data.Bytes;
+
+public class RowRange {
+ private final Bytes start;
+ private final Bytes end;
+
+ public RowRange(Bytes start, Bytes end) {
+ Objects.requireNonNull(start);
+ Objects.requireNonNull(end);
+ this.start = start;
+ this.end = end;
+ }
+
+ public Bytes getStart() {
+ return start;
+ }
+
+ public Bytes getEnd() {
+ return end;
+ }
+
+ @Override
+ public int hashCode() {
+ return start.hashCode() + 31 * end.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof RowRange) {
+ RowRange or = (RowRange) o;
+ return start.equals(or.start) && end.equals(or.end);
+ }
+
+ return false;
+ }
+
+ private static void encNonAscii(StringBuilder sb, Bytes bytes) {
+ if (bytes == null) {
+ sb.append("null");
+ } else {
+ for (int i = 0; i < bytes.length(); i++) {
+ byte b = bytes.byteAt(i);
+ if (b >= 32 && b <= 126 && b != '\\') {
+ sb.append((char) b);
+ } else {
+ sb.append(String.format("\\x%02x", b & 0xff));
+ }
+ }
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(");
+ encNonAscii(sb, start);
+ sb.append(", ");
+ encNonAscii(sb, end);
+ sb.append("]");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
new file mode 100644
index 0000000..9533a7a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/common/TransientRegistry.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.common;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.xml.bind.DatatypeConverter;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.api.data.Bytes;
+
+/**
+ * This class offers a standard way to register transient ranges. The project level documentation
+ * provides a comprehensive overview.
+ */
+
+public class TransientRegistry {
+
+ private SimpleConfiguration appConfig;
+ private static final String PREFIX = "recipes.transientRange.";
+
+ /**
+ * @param appConfig Fluo application config. Can be obtained from
+ * {@link FluoConfiguration#getAppConfiguration()} before initializing fluo when adding
+ * Transient ranges. After Fluo is initialized, app config can be obtained from
+ * {@link FluoClient#getAppConfiguration()} or
+ * {@link org.apache.fluo.api.observer.Observer.Context#getAppConfiguration()}
+ */
+ public TransientRegistry(SimpleConfiguration appConfig) {
+ this.appConfig = appConfig;
+ }
+
+ /**
+ * This method is expected to be called before Fluo is initialized to register transient ranges.
+ *
+ */
+ public void addTransientRange(String id, RowRange range) {
+ String start = DatatypeConverter.printHexBinary(range.getStart().toArray());
+ String end = DatatypeConverter.printHexBinary(range.getEnd().toArray());
+
+ appConfig.setProperty(PREFIX + id, start + ":" + end);
+ }
+
+ /**
+ * This method is expected to be called after Fluo is initialized to get the ranges that were
+ * registered before initialization.
+ */
+ public List<RowRange> getTransientRanges() {
+ List<RowRange> ranges = new ArrayList<>();
+ Iterator<String> keys = appConfig.getKeys(PREFIX.substring(0, PREFIX.length() - 1));
+ while (keys.hasNext()) {
+ String key = keys.next();
+ String val = appConfig.getString(key);
+ String[] sa = val.split(":");
+ RowRange rowRange =
+ new RowRange(Bytes.of(DatatypeConverter.parseHexBinary(sa[0])),
+ Bytes.of(DatatypeConverter.parseHexBinary(sa[1])));
+ ranges.add(rowRange);
+ }
+ return ranges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
new file mode 100644
index 0000000..e09bed5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/data/RowHasher.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.data;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.hash.Hashing;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.BytesBuilder;
+import org.apache.fluo.recipes.core.common.Pirtos;
+
+/**
+ * This recipe provides code to help add a hash of the row as a prefix of the row. Using this recipe
+ * rows are structured like the following.
+ *
+ * <p>
+ * {@code <prefix>:<fixed len row hash>:<user row>}
+ *
+ * <p>
+ * The recipe also provides code the help generate split points and configure balancing of the
+ * prefix.
+ *
+ * <p>
+ * The project documentation has more information.
+ */
+public class RowHasher {
+
+ private static final int HASH_LEN = 4;
+
+ public Pirtos getTableOptimizations(int numTablets) {
+
+ List<Bytes> splits = new ArrayList<>(numTablets - 1);
+
+ int numSplits = numTablets - 1;
+ int distance = (((int) Math.pow(Character.MAX_RADIX, HASH_LEN) - 1) / numTablets) + 1;
+ int split = distance;
+ for (int i = 0; i < numSplits; i++) {
+ splits.add(Bytes.of(prefix
+ + Strings.padStart(Integer.toString(split, Character.MAX_RADIX), HASH_LEN, '0')));
+ split += distance;
+ }
+
+ splits.add(Bytes.of(prefix + "~"));
+
+
+ Pirtos pirtos = new Pirtos();
+ pirtos.setSplits(splits);
+ pirtos.setTabletGroupingRegex(Pattern.quote(prefix.toString()));
+
+ return pirtos;
+ }
+
+
+ private Bytes prefix;
+
+ public RowHasher(String prefix) {
+ this.prefix = Bytes.of(prefix + ":");
+ }
+
+ /**
+ * @return Returns input with prefix and hash of input prepended.
+ */
+ public Bytes addHash(String row) {
+ return addHash(Bytes.of(row));
+ }
+
+ /**
+ * @return Returns input with prefix and hash of input prepended.
+ */
+ public Bytes addHash(Bytes row) {
+ BytesBuilder builder = Bytes.newBuilder(prefix.length() + 5 + row.length());
+ builder.append(prefix);
+ builder.append(genHash(row));
+ builder.append(":");
+ builder.append(row);
+ return builder.toBytes();
+ }
+
+ private boolean hasHash(Bytes row) {
+ for (int i = prefix.length(); i < prefix.length() + HASH_LEN; i++) {
+ byte b = row.byteAt(i);
+ boolean isAlphaNum = (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9');
+ if (!isAlphaNum) {
+ return false;
+ }
+ }
+
+ if (row.byteAt(prefix.length() - 1) != ':' || row.byteAt(prefix.length() + HASH_LEN) != ':') {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * @return Returns input with prefix and hash stripped from beginning.
+ */
+ public Bytes removeHash(Bytes row) {
+ Preconditions.checkArgument(row.length() >= prefix.length() + 5,
+ "Row is shorter than expected " + row);
+ Preconditions.checkArgument(row.subSequence(0, prefix.length()).equals(prefix),
+ "Row does not have expected prefix " + row);
+ Preconditions.checkArgument(hasHash(row), "Row does not have expected hash " + row);
+ return row.subSequence(prefix.length() + 5, row.length());
+ }
+
+ private static String genHash(Bytes row) {
+ int hash = Hashing.murmur3_32().hashBytes(row.toArray()).asInt();
+ hash = hash & 0x7fffffff;
+ // base 36 gives a lot more bins in 4 bytes than hex, but it is still human readable which is
+ // nice for debugging.
+ String hashString =
+ Strings.padStart(Integer.toString(hash, Character.MAX_RADIX), HASH_LEN, '0');
+ hashString = hashString.substring(hashString.length() - HASH_LEN);
+
+ return hashString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
new file mode 100644
index 0000000..4f14f65
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Export.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Objects;
+
+public class Export<K, V> {
+ private final K key;
+ private final V value;
+
+ public Export(K key, V val) {
+ Objects.requireNonNull(key);
+ Objects.requireNonNull(val);
+ this.key = key;
+ this.value = val;
+ }
+
+ public K getKey() {
+ return key;
+ }
+
+ public V getValue() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportBucket.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportBucket.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportBucket.java
new file mode 100644
index 0000000..cf6dfb4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportBucket.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import com.google.common.base.Preconditions;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumn;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.recipes.core.impl.BucketUtil;
+import org.apache.fluo.recipes.core.types.StringEncoder;
+import org.apache.fluo.recipes.core.types.TypeLayer;
+import org.apache.fluo.recipes.core.types.TypedTransactionBase;
+
+/**
+ * This class encapsulates a buckets serialization code.
+ */
+class ExportBucket {
+ private static final String NOTIFICATION_CF = "fluoRecipes";
+ private static final String NOTIFICATION_CQ_PREFIX = "eq:";
+ private static final Column EXPORT_COL = new Column("e", "v");
+ private static final Column NEXT_COL = new Column("e", "next");
+
+ static Column newNotificationColumn(String queueId) {
+ return new Column(NOTIFICATION_CF, NOTIFICATION_CQ_PREFIX + queueId);
+ }
+
+ private final TypedTransactionBase ttx;
+ private final String qid;
+ private final Bytes bucketRow;
+
+ static Bytes generateBucketRow(String qid, int bucket, int numBuckets) {
+ return Bytes.of(qid + ":" + BucketUtil.genBucketId(bucket, numBuckets));
+ }
+
+ ExportBucket(TransactionBase tx, String qid, int bucket, int numBuckets) {
+ // TODO encode in a more robust way... but for now fail early
+ Preconditions.checkArgument(!qid.contains(":"), "Export QID can not contain :");
+ this.ttx = new TypeLayer(new StringEncoder()).wrap(tx);
+ this.qid = qid;
+ this.bucketRow = generateBucketRow(qid, bucket, numBuckets);
+ }
+
+ ExportBucket(TransactionBase tx, Bytes bucketRow) {
+ this.ttx = new TypeLayer(new StringEncoder()).wrap(tx);
+
+ int colonLoc = -1;
+
+ for (int i = 0; i < bucketRow.length(); i++) {
+ if (bucketRow.byteAt(i) == ':') {
+ colonLoc = i;
+ break;
+ }
+ }
+
+ Preconditions.checkArgument(colonLoc != -1 && colonLoc != bucketRow.length(),
+ "Invalid bucket row " + bucketRow);
+ Preconditions.checkArgument(bucketRow.byteAt(bucketRow.length() - 1) == ':',
+ "Invalid bucket row " + bucketRow);
+
+ this.bucketRow = bucketRow.subSequence(0, bucketRow.length() - 1);
+ this.qid = bucketRow.subSequence(0, colonLoc).toString();
+ }
+
+ private static byte[] encSeq(long l) {
+ byte[] ret = new byte[8];
+ ret[0] = (byte) (l >>> 56);
+ ret[1] = (byte) (l >>> 48);
+ ret[2] = (byte) (l >>> 40);
+ ret[3] = (byte) (l >>> 32);
+ ret[4] = (byte) (l >>> 24);
+ ret[5] = (byte) (l >>> 16);
+ ret[6] = (byte) (l >>> 8);
+ ret[7] = (byte) (l >>> 0);
+ return ret;
+ }
+
+ private static long decodeSeq(Bytes seq) {
+ return (((long) seq.byteAt(0) << 56) + ((long) (seq.byteAt(1) & 255) << 48)
+ + ((long) (seq.byteAt(2) & 255) << 40) + ((long) (seq.byteAt(3) & 255) << 32)
+ + ((long) (seq.byteAt(4) & 255) << 24) + ((seq.byteAt(5) & 255) << 16)
+ + ((seq.byteAt(6) & 255) << 8) + ((seq.byteAt(7) & 255) << 0));
+ }
+
+
+ public void add(long seq, byte[] key, byte[] value) {
+ Bytes row =
+ Bytes.newBuilder(bucketRow.length() + 1 + key.length + 8).append(bucketRow).append(":")
+ .append(key).append(encSeq(seq)).toBytes();
+ ttx.set(row, EXPORT_COL, Bytes.of(value));
+ }
+
+ /**
+ * Computes the minimial row for a bucket
+ */
+ private Bytes getMinimalRow() {
+ return Bytes.newBuilder(bucketRow.length() + 1).append(bucketRow).append(":").toBytes();
+ }
+
+ public void notifyExportObserver() {
+ ttx.mutate().row(getMinimalRow()).col(newNotificationColumn(qid)).weaklyNotify();
+ }
+
+ public Iterator<ExportEntry> getExportIterator(Bytes continueRow) {
+ ScannerConfiguration sc = new ScannerConfiguration();
+
+ if (continueRow != null) {
+ Span tmpSpan = Span.prefix(bucketRow);
+ Span nextSpan =
+ new Span(new RowColumn(continueRow, EXPORT_COL), true, tmpSpan.getEnd(),
+ tmpSpan.isEndInclusive());
+ sc.setSpan(nextSpan);
+ } else {
+ sc.setSpan(Span.prefix(bucketRow));
+ }
+
+ sc.fetchColumn(EXPORT_COL.getFamily(), EXPORT_COL.getQualifier());
+ RowIterator iter = ttx.get(sc);
+
+ if (iter.hasNext()) {
+ return new ExportIterator(iter);
+ } else {
+ return Collections.<ExportEntry>emptySet().iterator();
+ }
+ }
+
+ private class ExportIterator implements Iterator<ExportEntry> {
+
+ private RowIterator rowIter;
+ private Bytes lastRow;
+
+ public ExportIterator(RowIterator rowIter) {
+ this.rowIter = rowIter;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return rowIter.hasNext();
+ }
+
+ @Override
+ public ExportEntry next() {
+ Entry<Bytes, ColumnIterator> rowCol = rowIter.next();
+ Bytes row = rowCol.getKey();
+
+ Bytes keyBytes = row.subSequence(bucketRow.length() + 1, row.length() - 8);
+ Bytes seqBytes = row.subSequence(row.length() - 8, row.length());
+
+ ExportEntry ee = new ExportEntry();
+
+ ee.key = keyBytes.toArray();
+ ee.seq = decodeSeq(seqBytes);
+ // TODO maybe leave as Bytes?
+ ee.value = rowCol.getValue().next().getValue().toArray();
+
+ lastRow = row;
+
+ return ee;
+ }
+
+ @Override
+ public void remove() {
+ ttx.mutate().row(lastRow).col(EXPORT_COL).delete();
+ }
+ }
+
+ public Bytes getContinueRow() {
+ return ttx.get(getMinimalRow(), NEXT_COL);
+ }
+
+ public void setContinueRow(ExportEntry ee) {
+ Bytes nextRow =
+ Bytes.newBuilder(bucketRow.length() + 1 + ee.key.length + 8).append(bucketRow).append(":")
+ .append(ee.key).append(encSeq(ee.seq)).toBytes();
+
+ ttx.set(getMinimalRow(), NEXT_COL, nextRow);
+ }
+
+ public void clearContinueRow() {
+ ttx.delete(getMinimalRow(), NEXT_COL);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportEntry.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportEntry.java
new file mode 100644
index 0000000..e680204
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportEntry.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+class ExportEntry {
+ byte[] key;
+ long seq;
+ byte[] value;
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
new file mode 100644
index 0000000..940575b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportObserver.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.Iterators;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.observer.AbstractObserver;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+public class ExportObserver<K, V> extends AbstractObserver {
+
+ private static class MemLimitIterator implements Iterator<ExportEntry> {
+
+ private long memConsumed = 0;
+ private long memLimit;
+ private int extraPerKey;
+ private Iterator<ExportEntry> source;
+
+ public MemLimitIterator(Iterator<ExportEntry> input, long limit, int extraPerKey) {
+ this.source = input;
+ this.memLimit = limit;
+ this.extraPerKey = extraPerKey;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return memConsumed < memLimit && source.hasNext();
+ }
+
+ @Override
+ public ExportEntry next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ ExportEntry ee = source.next();
+ memConsumed += ee.key.length + extraPerKey + ee.value.length;
+ return ee;
+ }
+
+ @Override
+ public void remove() {
+ source.remove();
+ }
+ }
+
+ private String queueId;
+ private Class<K> keyType;
+ private Class<V> valType;
+ SimpleSerializer serializer;
+ private Exporter<K, V> exporter;
+
+ private long memLimit;
+
+ protected String getQueueId() {
+ return queueId;
+ }
+
+ SimpleSerializer getSerializer() {
+ return serializer;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(Context context) throws Exception {
+ queueId = context.getParameters().get("queueId");
+ ExportQueue.Options opts = new ExportQueue.Options(queueId, context.getAppConfiguration());
+
+ // TODO defer loading classes... so that not done during fluo init
+ // TODO move class loading to centralized place... also attempt to check type params
+ keyType = (Class<K>) getClass().getClassLoader().loadClass(opts.keyType);
+ valType = (Class<V>) getClass().getClassLoader().loadClass(opts.valueType);
+ exporter =
+ getClass().getClassLoader().loadClass(opts.exporterType).asSubclass(Exporter.class)
+ .newInstance();
+
+ serializer = SimpleSerializer.getInstance(context.getAppConfiguration());
+
+ memLimit = opts.getBufferSize();
+
+ exporter.init(queueId, context);
+ }
+
+ @Override
+ public ObservedColumn getObservedColumn() {
+ return new ObservedColumn(ExportBucket.newNotificationColumn(queueId), NotificationType.WEAK);
+ }
+
+ @Override
+ public void process(TransactionBase tx, Bytes row, Column column) throws Exception {
+ ExportBucket bucket = new ExportBucket(tx, row);
+
+ Bytes continueRow = bucket.getContinueRow();
+
+ Iterator<ExportEntry> input = bucket.getExportIterator(continueRow);
+ MemLimitIterator memLimitIter = new MemLimitIterator(input, memLimit, 8 + queueId.length());
+
+ Iterator<SequencedExport<K, V>> exportIterator =
+ Iterators.transform(
+ memLimitIter,
+ ee -> new SequencedExport<>(serializer.deserialize(ee.key, keyType), serializer
+ .deserialize(ee.value, valType), ee.seq));
+
+ exportIterator = Iterators.consumingIterator(exportIterator);
+
+ exporter.processExports(exportIterator);
+
+ if (input.hasNext()) {
+ // not everything was processed so notify self
+ bucket.notifyExportObserver();
+
+ if (!memLimitIter.hasNext()) {
+ // stopped because of mem limit... set continue key
+ bucket.setContinueRow(input.next());
+ continueRow = null;
+ }
+ }
+
+ if (continueRow != null) {
+ bucket.clearContinueRow();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
new file mode 100644
index 0000000..f013872
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/ExportQueue.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import com.google.common.hash.Hashing;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.ObserverConfiguration;
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.recipes.core.common.Pirtos;
+import org.apache.fluo.recipes.core.common.RowRange;
+import org.apache.fluo.recipes.core.common.TransientRegistry;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+public class ExportQueue<K, V> {
+
+ private static final String RANGE_BEGIN = "#";
+ private static final String RANGE_END = ":~";
+
+ private int numBuckets;
+ private SimpleSerializer serializer;
+ private String queueId;
+
+ // usage hint : could be created once in an observers init method
+ // usage hint : maybe have a queue for each type of data being exported???
+ // maybe less queues are
+ // more efficient though because more batching at export time??
+ ExportQueue(Options opts, SimpleSerializer serializer) throws Exception {
+ // TODO sanity check key type based on type params
+ // TODO defer creating classes until needed.. so that its not done during Fluo init
+ this.queueId = opts.queueId;
+ this.numBuckets = opts.numBuckets;
+ this.serializer = serializer;
+ }
+
+ public void add(TransactionBase tx, K key, V value) {
+ addAll(tx, Collections.singleton(new Export<>(key, value)).iterator());
+ }
+
+ public void addAll(TransactionBase tx, Iterator<Export<K, V>> exports) {
+
+ Set<Integer> bucketsNotified = new HashSet<>();
+ while (exports.hasNext()) {
+ Export<K, V> export = exports.next();
+
+ byte[] k = serializer.serialize(export.getKey());
+ byte[] v = serializer.serialize(export.getValue());
+
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ int bucketId = Math.abs(hash % numBuckets);
+
+ ExportBucket bucket = new ExportBucket(tx, queueId, bucketId, numBuckets);
+ bucket.add(tx.getStartTimestamp(), k, v);
+
+ if (!bucketsNotified.contains(bucketId)) {
+ bucket.notifyExportObserver();
+ bucketsNotified.add(bucketId);
+ }
+ }
+ }
+
+ public static <K2, V2> ExportQueue<K2, V2> getInstance(String exportQueueId,
+ SimpleConfiguration appConfig) {
+ Options opts = new Options(exportQueueId, appConfig);
+ try {
+ return new ExportQueue<>(opts, SimpleSerializer.getInstance(appConfig));
+ } catch (Exception e) {
+ // TODO
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Call this method before initializing Fluo.
+ *
+ * @param fluoConfig The configuration that will be used to initialize fluo.
+ */
+ public static void configure(FluoConfiguration fluoConfig, Options opts) {
+ SimpleConfiguration appConfig = fluoConfig.getAppConfiguration();
+ opts.save(appConfig);
+
+ fluoConfig.addObserver(new ObserverConfiguration(ExportObserver.class.getName())
+ .setParameters(Collections.singletonMap("queueId", opts.queueId)));
+
+ Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
+ Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
+
+ new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("exportQueue."
+ + opts.queueId, new RowRange(exportRangeStart, exportRangeStop));
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for all previously configured export queues.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
+
+ public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
+ HashSet<String> queueIds = new HashSet<>();
+ appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
+ k -> queueIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
+
+ Pirtos pirtos = new Pirtos();
+ queueIds.forEach(qid -> pirtos.merge(getTableOptimizations(qid, appConfig)));
+
+ return pirtos;
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for the specified export queue.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
+ public static Pirtos getTableOptimizations(String queueId, SimpleConfiguration appConfig) {
+ Options opts = new Options(queueId, appConfig);
+
+ List<Bytes> splits = new ArrayList<>();
+
+ Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
+ Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
+
+ splits.add(exportRangeStart);
+ splits.add(exportRangeStop);
+
+ List<Bytes> exportSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ exportSplits.add(ExportBucket.generateBucketRow(opts.queueId, i, opts.numBuckets));
+ }
+ Collections.sort(exportSplits);
+ splits.addAll(exportSplits);
+
+ Pirtos pirtos = new Pirtos();
+ pirtos.setSplits(splits);
+
+ // the tablet with end row <queueId># does not contain any data for the export queue and
+ // should not be grouped with the export queue
+ pirtos.setTabletGroupingRegex(Pattern.quote(queueId + ":"));
+
+ return pirtos;
+ }
+
+ public static class Options {
+
+ private static final String PREFIX = "recipes.exportQueue.";
+ static final long DEFAULT_BUFFER_SIZE = 1 << 20;
+ static final int DEFAULT_BUCKETS_PER_TABLET = 10;
+
+ int numBuckets;
+ Integer bucketsPerTablet = null;
+ Long bufferSize;
+
+ String keyType;
+ String valueType;
+ String exporterType;
+ String queueId;
+
+ Options(String queueId, SimpleConfiguration appConfig) {
+ this.queueId = queueId;
+
+ this.numBuckets = appConfig.getInt(PREFIX + queueId + ".buckets");
+ this.exporterType = appConfig.getString(PREFIX + queueId + ".exporter");
+ this.keyType = appConfig.getString(PREFIX + queueId + ".key");
+ this.valueType = appConfig.getString(PREFIX + queueId + ".val");
+ this.bufferSize = appConfig.getLong(PREFIX + queueId + ".bufferSize", DEFAULT_BUFFER_SIZE);
+ this.bucketsPerTablet =
+ appConfig.getInt(PREFIX + queueId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
+ }
+
+ public Options(String queueId, String exporterType, String keyType, String valueType,
+ int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+
+ this.queueId = queueId;
+ this.numBuckets = buckets;
+ this.exporterType = exporterType;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+
+ public <K, V> Options(String queueId, Class<? extends Exporter<K, V>> exporter,
+ Class<K> keyType, Class<V> valueType, int buckets) {
+ this(queueId, exporter.getName(), keyType.getName(), valueType.getName(), buckets);
+ }
+
+ /**
+ * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
+ * be used to actually deserialize and process the updates. This limit does not account for
+ * object overhead in java, which can be significant.
+ *
+ * <p>
+ * The way memory read is calculated is by summing the length of serialized key and value byte
+ * arrays. Once this sum exceeds the configured memory limit, no more export key values are
+ * processed in the current transaction. When not everything is processed, the observer
+ * processing exports will notify itself causing another transaction to continue processing
+ * later.
+ */
+ public Options setBufferSize(long bufferSize) {
+ Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
+ this.bufferSize = bufferSize;
+ return this;
+ }
+
+ long getBufferSize() {
+ if (bufferSize == null) {
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ return bufferSize;
+ }
+
+ /**
+ * Sets the number of buckets per tablet to generate. This affects how many split points will be
+ * generated when optimizing the Accumulo table.
+ *
+ */
+ public Options setBucketsPerTablet(int bucketsPerTablet) {
+ Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
+ + bucketsPerTablet);
+ this.bucketsPerTablet = bucketsPerTablet;
+ return this;
+ }
+
+ int getBucketsPerTablet() {
+ if (bucketsPerTablet == null) {
+ return DEFAULT_BUCKETS_PER_TABLET;
+ }
+
+ return bucketsPerTablet;
+ }
+
+ void save(SimpleConfiguration appConfig) {
+ appConfig.setProperty(PREFIX + queueId + ".buckets", numBuckets + "");
+ appConfig.setProperty(PREFIX + queueId + ".exporter", exporterType + "");
+ appConfig.setProperty(PREFIX + queueId + ".key", keyType);
+ appConfig.setProperty(PREFIX + queueId + ".val", valueType);
+
+ if (bufferSize != null) {
+ appConfig.setProperty(PREFIX + queueId + ".bufferSize", bufferSize);
+ }
+ if (bucketsPerTablet != null) {
+ appConfig.setProperty(PREFIX + queueId + ".bucketsPerTablet", bucketsPerTablet);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
new file mode 100644
index 0000000..529d5f3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/Exporter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+import java.util.Iterator;
+
+import org.apache.fluo.api.observer.Observer.Context;
+
+public abstract class Exporter<K, V> {
+
+ public void init(String queueId, Context observerContext) throws Exception {}
+
+ /**
+ * Must be able to handle same key being exported multiple times and key being exported out of
+ * order. The sequence number is meant to help with this.
+ *
+ * <p>
+ * If multiple export entries with the same key are passed in, then the entries with the same key
+ * will be consecutive and in ascending sequence order.
+ *
+ * <p>
+ * If the call to process exports is unexpectedly terminated, it will be called again later with
+ * at least the same data. For example suppose an exporter was passed the following entries.
+ *
+ * <ul>
+ * <li>key=0 sequence=9 value=abc
+ * <li>key=1 sequence=13 value=d
+ * <li>key=1 sequence=17 value=e
+ * <li>key=1 sequence=23 value=f
+ * <li>key=2 sequence=19 value=x
+ * </ul>
+ *
+ * <p>
+ * Assume the exporter exports some of these and then fails before completing all of them. The
+ * next time its called it will be passed what it saw before, but it could also be passed more.
+ *
+ * <ul>
+ * <li>key=0 sequence=9 value=abc
+ * <li>key=1 sequence=13 value=d
+ * <li>key=1 sequence=17 value=e
+ * <li>key=1 sequence=23 value=f
+ * <li>key=1 sequence=29 value=g
+ * <li>key=2 sequence=19 value=x
+ * <li>key=2 sequence=77 value=y
+ * </ul>
+ *
+ */
+ protected abstract void processExports(Iterator<SequencedExport<K, V>> exports);
+
+ // TODO add close
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
new file mode 100644
index 0000000..ef1cfe2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/export/SequencedExport.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.export;
+
+public class SequencedExport<K, V> extends Export<K, V> {
+ private final long seq;
+
+ SequencedExport(K k, V v, long seq) {
+ super(k, v);
+ this.seq = seq;
+ }
+
+ public long getSequence() {
+ return seq;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/impl/BucketUtil.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/impl/BucketUtil.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/impl/BucketUtil.java
new file mode 100644
index 0000000..06bdfa8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/impl/BucketUtil.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.impl;
+
+public class BucketUtil {
+ public static String genBucketId(int bucket, int maxBucket) {
+ int bucketLen = Integer.toHexString(maxBucket).length();
+ // TODO printf is slow
+ return String.format("%0" + bucketLen + "x", bucket);
+ }
+}
[04/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMapObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMapObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMapObserver.java
deleted file mode 100644
index 6d29d17..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMapObserver.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.observer.AbstractObserver;
-
-/**
- * This class is configured for use by CollisionFreeMap.configure(FluoConfiguration,
- * CollisionFreeMap.Options) . This class should never have to be used directly.
- */
-
-public class CollisionFreeMapObserver extends AbstractObserver {
-
- @SuppressWarnings("rawtypes")
- private CollisionFreeMap cfm;
- private String mapId;
-
- public CollisionFreeMapObserver() {}
-
- @Override
- public void init(Context context) throws Exception {
- this.mapId = context.getParameters().get("mapId");
- cfm = CollisionFreeMap.getInstance(mapId, context.getAppConfiguration());
- cfm.updateObserver.init(mapId, context);
- }
-
- @Override
- public void process(TransactionBase tx, Bytes row, Column col) throws Exception {
- cfm.process(tx, row, col);
- }
-
- @Override
- public ObservedColumn getObservedColumn() {
- // TODO constants
- return new ObservedColumn(new Column("fluoRecipes", "cfm:" + mapId), NotificationType.WEAK);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/Combiner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/Combiner.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/Combiner.java
deleted file mode 100644
index 776580f..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/Combiner.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Iterator;
-import java.util.Optional;
-
-public interface Combiner<K, V> {
- /**
- * This function is called to combine the current value of a key with updates that were queued for
- * the key. See the collision free map project level documentation for more information.
- *
- * @return Then new value for the key. Returning Optional.absent() will cause the key to be
- * deleted.
- */
-
- Optional<V> combine(K key, Iterator<V> updates);
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/NullUpdateObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/NullUpdateObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/NullUpdateObserver.java
deleted file mode 100644
index 626a210..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/NullUpdateObserver.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Iterator;
-
-import org.apache.fluo.api.client.TransactionBase;
-
-class NullUpdateObserver<K, V> extends UpdateObserver<K, V> {
- @Override
- public void updatingValues(TransactionBase tx, Iterator<Update<K, V>> updates) {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/Update.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/Update.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/Update.java
deleted file mode 100644
index cb34f1e..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/Update.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Optional;
-
-public class Update<K, V> {
-
- private final K key;
- private final Optional<V> oldValue;
- private final Optional<V> newValue;
-
- Update(K key, Optional<V> oldValue, Optional<V> newValue) {
- this.key = key;
- this.oldValue = oldValue;
- this.newValue = newValue;
- }
-
- public K getKey() {
- return key;
- }
-
- public Optional<V> getNewValue() {
- return newValue;
- }
-
- public Optional<V> getOldValue() {
- return oldValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/UpdateObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/UpdateObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/UpdateObserver.java
deleted file mode 100644
index 4cb6185..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/UpdateObserver.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.util.Iterator;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.observer.Observer.Context;
-
-/**
- * A {@link CollisionFreeMap} calls this to allow additional processing to be done when key values
- * are updated. See the project level documentation for more information.
- */
-
-public abstract class UpdateObserver<K, V> {
- public void init(String mapId, Context observerContext) throws Exception {}
-
- public abstract void updatingValues(TransactionBase tx, Iterator<Update<K, V>> updates);
-
- // TODO add close
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/serialization/SimpleSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/serialization/SimpleSerializer.java b/modules/core/src/main/java/org/apache/fluo/recipes/serialization/SimpleSerializer.java
deleted file mode 100644
index 0d823e2..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/serialization/SimpleSerializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.serialization;
-
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.SimpleConfiguration;
-
-public interface SimpleSerializer {
-
- /**
- * Called immediately after construction and passed Fluo application configuration.
- */
- public void init(SimpleConfiguration appConfig);
-
- // TODO refactor to support reuse of objects and byte arrays???
- public <T> byte[] serialize(T obj);
-
- public <T> T deserialize(byte[] serObj, Class<T> clazz);
-
- public static void setSetserlializer(FluoConfiguration fluoConfig,
- Class<? extends SimpleSerializer> serializerType) {
- setSetserlializer(fluoConfig, serializerType.getName());
- }
-
- public static void setSetserlializer(FluoConfiguration fluoConfig, String serializerType) {
- fluoConfig.getAppConfiguration().setProperty("recipes.serializer", serializerType);
- }
-
- public static SimpleSerializer getInstance(SimpleConfiguration appConfig) {
- String serType =
- appConfig.getString("recipes.serializer",
- "org.apache.fluo.recipes.kryo.KryoSimplerSerializer");
- try {
- SimpleSerializer simplerSer =
- SimpleSerializer.class.getClassLoader().loadClass(serType)
- .asSubclass(SimpleSerializer.class).newInstance();
- simplerSer.init(appConfig);
- return simplerSer;
- } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/transaction/LogEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/LogEntry.java b/modules/core/src/main/java/org/apache/fluo/recipes/transaction/LogEntry.java
deleted file mode 100644
index 925d6ab..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/LogEntry.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.transaction;
-
-import java.util.Objects;
-
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-
-/**
- * Logs an operation (i.e GET, SET, or DELETE) in a Transaction. Multiple LogEntry objects make up a
- * {@link TxLog}.
- */
-public class LogEntry {
-
- public enum Operation {
- GET, SET, DELETE
- }
-
- private Operation op;
- private Bytes row;
- private Column col;
- private Bytes value;
-
- private LogEntry() {}
-
- private LogEntry(Operation op, Bytes row, Column col, Bytes value) {
- Objects.requireNonNull(op);
- Objects.requireNonNull(row);
- Objects.requireNonNull(col);
- Objects.requireNonNull(value);
- this.op = op;
- this.row = row;
- this.col = col;
- this.value = value;
- }
-
- public Operation getOp() {
- return op;
- }
-
- public Bytes getRow() {
- return row;
- }
-
- public Column getColumn() {
- return col;
- }
-
- public Bytes getValue() {
- return value;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof LogEntry) {
- LogEntry other = (LogEntry) o;
- return ((op == other.op) && row.equals(other.row) && col.equals(other.col) && value
- .equals(other.value));
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- int result = op.hashCode();
- result = 31 * result + row.hashCode();
- result = 31 * result + col.hashCode();
- result = 31 * result + value.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "LogEntry{op=" + op + ", row=" + row + ", col=" + col + ", value=" + value + "}";
- }
-
- public static LogEntry newGet(String row, Column col, String value) {
- return newGet(Bytes.of(row), col, Bytes.of(value));
- }
-
- public static LogEntry newGet(Bytes row, Column col, Bytes value) {
- return new LogEntry(Operation.GET, row, col, value);
- }
-
- public static LogEntry newSet(String row, Column col, String value) {
- return newSet(Bytes.of(row), col, Bytes.of(value));
- }
-
- public static LogEntry newSet(Bytes row, Column col, Bytes value) {
- return new LogEntry(Operation.SET, row, col, value);
- }
-
- public static LogEntry newDelete(String row, Column col) {
- return newDelete(Bytes.of(row), col);
- }
-
- public static LogEntry newDelete(Bytes row, Column col) {
- return new LogEntry(Operation.DELETE, row, col, Bytes.EMPTY);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransaction.java b/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransaction.java
deleted file mode 100644
index 48b4ad8..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransaction.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.transaction;
-
-import java.util.function.Predicate;
-
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.exceptions.CommitException;
-
-/**
- * An implementation of {@link Transaction} that logs all transactions operations (GET, SET, or
- * DELETE) in a {@link TxLog} that can be used for exports
- */
-public class RecordingTransaction extends RecordingTransactionBase implements Transaction {
-
- private final Transaction tx;
-
- private RecordingTransaction(Transaction tx) {
- super(tx);
- this.tx = tx;
- }
-
- private RecordingTransaction(Transaction tx, Predicate<LogEntry> filter) {
- super(tx, filter);
- this.tx = tx;
- }
-
- @Override
- public void commit() throws CommitException {
- tx.commit();
- }
-
- @Override
- public void close() {
- tx.close();
- }
-
- /**
- * Creates a RecordingTransaction by wrapping an existing Transaction
- */
- public static RecordingTransaction wrap(Transaction tx) {
- return new RecordingTransaction(tx);
- }
-
- /**
- * Creates a RecordingTransaction using the provided LogEntry filter and existing Transaction
- */
- public static RecordingTransaction wrap(Transaction tx, Predicate<LogEntry> filter) {
- return new RecordingTransaction(tx, filter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransactionBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransactionBase.java
deleted file mode 100644
index d00718a..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/RecordingTransactionBase.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.transaction;
-
-import java.util.AbstractMap;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Predicate;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumn;
-import org.apache.fluo.api.exceptions.AlreadySetException;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-
-/**
- * An implementation of {@link TransactionBase} that logs all transactions operations (GET, SET, or
- * DELETE) in a {@link TxLog} that can be used for exports
- */
-public class RecordingTransactionBase implements TransactionBase {
-
- private final TransactionBase txb;
- private final TxLog txLog = new TxLog();
- private final Predicate<LogEntry> filter;
-
- RecordingTransactionBase(TransactionBase txb, Predicate<LogEntry> filter) {
- this.txb = txb;
- this.filter = filter;
- }
-
- RecordingTransactionBase(TransactionBase txb) {
- this(txb, le -> true);
- }
-
- @Override
- public void setWeakNotification(Bytes row, Column col) {
- txb.setWeakNotification(row, col);
- }
-
- @Override
- public void setWeakNotification(String row, Column col) {
- txb.setWeakNotification(row, col);
- }
-
- @Override
- public void set(Bytes row, Column col, Bytes value) throws AlreadySetException {
- txLog.filteredAdd(LogEntry.newSet(row, col, value), filter);
- txb.set(row, col, value);
- }
-
- @Override
- public void set(String row, Column col, String value) throws AlreadySetException {
- txLog.filteredAdd(LogEntry.newSet(row, col, value), filter);
- txb.set(row, col, value);
- }
-
- @Override
- public void delete(Bytes row, Column col) {
- txLog.filteredAdd(LogEntry.newDelete(row, col), filter);
- txb.delete(row, col);
- }
-
- @Override
- public void delete(String row, Column col) {
- txLog.filteredAdd(LogEntry.newDelete(row, col), filter);
- txb.delete(row, col);
- }
-
- /**
- * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
- */
- @Override
- public Bytes get(Bytes row, Column col) {
- Bytes val = txb.get(row, col);
- if (val != null) {
- txLog.filteredAdd(LogEntry.newGet(row, col, val), filter);
- }
- return val;
- }
-
- /**
- * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
- */
- @Override
- public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
- Map<Column, Bytes> colVal = txb.get(row, columns);
- for (Map.Entry<Column, Bytes> entry : colVal.entrySet()) {
- txLog.filteredAdd(LogEntry.newGet(row, entry.getKey(), entry.getValue()), filter);
- }
- return colVal;
- }
-
- /**
- * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
- */
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
- Map<Bytes, Map<Column, Bytes>> rowColVal = txb.get(rows, columns);
- for (Map.Entry<Bytes, Map<Column, Bytes>> rowEntry : rowColVal.entrySet()) {
- for (Map.Entry<Column, Bytes> colEntry : rowEntry.getValue().entrySet()) {
- txLog.filteredAdd(
- LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
- }
- }
- return rowColVal;
- }
-
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
- Map<Bytes, Map<Column, Bytes>> rowColVal = txb.get(rowColumns);
- for (Map.Entry<Bytes, Map<Column, Bytes>> rowEntry : rowColVal.entrySet()) {
- for (Map.Entry<Column, Bytes> colEntry : rowEntry.getValue().entrySet()) {
- txLog.filteredAdd(
- LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
- }
- }
- return rowColVal;
- }
-
- /**
- * Logs GETs for Row/Columns returned by iterators. Requests that return no data will not be
- * logged.
- */
- @Override
- public RowIterator get(ScannerConfiguration config) {
- final RowIterator rowIter = txb.get(config);
- if (rowIter != null) {
- return new RowIterator() {
-
- @Override
- public boolean hasNext() {
- return rowIter.hasNext();
- }
-
- @Override
- public Map.Entry<Bytes, ColumnIterator> next() {
- final Map.Entry<Bytes, ColumnIterator> rowEntry = rowIter.next();
- if ((rowEntry != null) && (rowEntry.getValue() != null)) {
- final ColumnIterator colIter = rowEntry.getValue();
- return new AbstractMap.SimpleEntry<>(rowEntry.getKey(), new ColumnIterator() {
-
- @Override
- public boolean hasNext() {
- return colIter.hasNext();
- }
-
- @Override
- public Map.Entry<Column, Bytes> next() {
- Map.Entry<Column, Bytes> colEntry = colIter.next();
- if (colEntry != null) {
- txLog.filteredAdd(
- LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()),
- filter);
- }
- return colEntry;
- }
- });
- }
- return rowEntry;
- }
- };
- }
- return rowIter;
- }
-
- @Override
- public long getStartTimestamp() {
- return txb.getStartTimestamp();
- }
-
- public TxLog getTxLog() {
- return txLog;
- }
-
- /**
- * Creates a RecordingTransactionBase by wrapping an existing TransactionBase
- */
- public static RecordingTransactionBase wrap(TransactionBase txb) {
- return new RecordingTransactionBase(txb);
- }
-
- /**
- * Creates a RecordingTransactionBase using the provided LogEntry filter function and existing
- * TransactionBase
- */
- public static RecordingTransactionBase wrap(TransactionBase txb, Predicate<LogEntry> filter) {
- return new RecordingTransactionBase(txb, filter);
- }
-
- @Override
- public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
- Map<String, Map<Column, String>> rowColVal = txb.gets(rowColumns);
- for (Map.Entry<String, Map<Column, String>> rowEntry : rowColVal.entrySet()) {
- for (Map.Entry<Column, String> colEntry : rowEntry.getValue().entrySet()) {
- txLog.filteredAdd(
- LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
- }
- }
- return rowColVal;
- }
-
- // TODO alot of these String methods may be more efficient if called the Byte version in this
- // class... this would avoid conversion from Byte->String->Byte
- @Override
- public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
- Map<String, Map<Column, String>> rowColVal = txb.gets(rows, columns);
- for (Map.Entry<String, Map<Column, String>> rowEntry : rowColVal.entrySet()) {
- for (Map.Entry<Column, String> colEntry : rowEntry.getValue().entrySet()) {
- txLog.filteredAdd(
- LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
- }
- }
- return rowColVal;
- }
-
- @Override
- public String gets(String row, Column col) {
- String val = txb.gets(row, col);
- if (val != null) {
- txLog.filteredAdd(LogEntry.newGet(row, col, val), filter);
- }
- return val;
- }
-
- @Override
- public Map<Column, String> gets(String row, Set<Column> columns) {
- Map<Column, String> colVal = txb.gets(row, columns);
- for (Map.Entry<Column, String> entry : colVal.entrySet()) {
- txLog.filteredAdd(LogEntry.newGet(row, entry.getKey(), entry.getValue()), filter);
- }
- return colVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/transaction/TxLog.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/TxLog.java b/modules/core/src/main/java/org/apache/fluo/recipes/transaction/TxLog.java
deleted file mode 100644
index 6f13891..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/transaction/TxLog.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.transaction;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Predicate;
-
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.RowColumn;
-
-/**
- * Contains list of operations (GET, SET, DELETE) performed during a {@link RecordingTransaction}
- */
-public class TxLog {
-
- private List<LogEntry> logEntries = new ArrayList<>();
-
- public TxLog() {}
-
- /**
- * Adds LogEntry to TxLog
- */
- public void add(LogEntry entry) {
- logEntries.add(entry);
- }
-
- /**
- * Adds LogEntry to TxLog if it passes filter
- */
- public void filteredAdd(LogEntry entry, Predicate<LogEntry> filter) {
- if (filter.test(entry)) {
- add(entry);
- }
- }
-
- /**
- * Returns all LogEntry in TxLog
- */
- public List<LogEntry> getLogEntries() {
- return Collections.unmodifiableList(logEntries);
- }
-
- /**
- * Returns true if TxLog is empty
- */
- public boolean isEmpty() {
- return logEntries.isEmpty();
- }
-
- /**
- * Returns a map of RowColumn changes given an operation
- */
- public Map<RowColumn, Bytes> getOperationMap(LogEntry.Operation op) {
- Map<RowColumn, Bytes> opMap = new HashMap<>();
- for (LogEntry entry : logEntries) {
- if (entry.getOp().equals(op)) {
- opMap.put(new RowColumn(entry.getRow(), entry.getColumn()), entry.getValue());
- }
- }
- return opMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/Encoder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/Encoder.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/Encoder.java
deleted file mode 100644
index 6b1b626..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/Encoder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.data.Bytes;
-
-/**
- * Transforms Java primitives to and from bytes using desired encoding
- *
- * @since 1.0.0
- */
-public interface Encoder {
-
- /**
- * Encodes an integer to {@link Bytes}
- */
- Bytes encode(int i);
-
- /**
- * Encodes a long to {@link Bytes}
- */
- Bytes encode(long l);
-
- /**
- * Encodes a String to {@link Bytes}
- */
- Bytes encode(String s);
-
- /**
- * Encodes a float to {@link Bytes}
- */
- Bytes encode(float f);
-
- /**
- * Encodes a double to {@link Bytes}
- */
- Bytes encode(double d);
-
- /**
- * Encodes a boolean to {@link Bytes}
- */
- Bytes encode(boolean b);
-
- /**
- * Decodes an integer from {@link Bytes}
- */
- int decodeInteger(Bytes b);
-
- /**
- * Decodes a long from {@link Bytes}
- */
- long decodeLong(Bytes b);
-
- /**
- * Decodes a String from {@link Bytes}
- */
- String decodeString(Bytes b);
-
- /**
- * Decodes a float from {@link Bytes}
- */
- float decodeFloat(Bytes b);
-
- /**
- * Decodes a double from {@link Bytes}
- */
- double decodeDouble(Bytes b);
-
- /**
- * Decodes a boolean from {@link Bytes}
- */
- boolean decodeBoolean(Bytes b);
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/StringEncoder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/StringEncoder.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/StringEncoder.java
deleted file mode 100644
index 10524ed..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/StringEncoder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.data.Bytes;
-
-/**
- * Transforms Java primitives to and from bytes using a String encoding
- *
- * @since 1.0.0
- */
-public class StringEncoder implements Encoder {
-
- @Override
- public Bytes encode(int i) {
- return encode(Integer.toString(i));
- }
-
- @Override
- public Bytes encode(long l) {
- return encode(Long.toString(l));
- }
-
- @Override
- public Bytes encode(String s) {
- return Bytes.of(s);
- }
-
- @Override
- public Bytes encode(float f) {
- return encode(Float.toString(f));
- }
-
- @Override
- public Bytes encode(double d) {
- return encode(Double.toString(d));
- }
-
- @Override
- public Bytes encode(boolean b) {
- return encode(Boolean.toString(b));
- }
-
- @Override
- public int decodeInteger(Bytes b) {
- return Integer.parseInt(decodeString(b));
- }
-
- @Override
- public long decodeLong(Bytes b) {
- return Long.parseLong(decodeString(b));
- }
-
- @Override
- public String decodeString(Bytes b) {
- return b.toString();
- }
-
- @Override
- public float decodeFloat(Bytes b) {
- return Float.parseFloat(decodeString(b));
- }
-
- @Override
- public double decodeDouble(Bytes b) {
- return Double.parseDouble(decodeString(b));
- }
-
- @Override
- public boolean decodeBoolean(Bytes b) {
- return Boolean.parseBoolean(decodeString(b));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypeLayer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypeLayer.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypeLayer.java
deleted file mode 100644
index c9c0e16..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypeLayer.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.nio.ByteBuffer;
-
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-
-/**
- * A simple convenience layer for Fluo. This layer attempts to make the following common operations
- * easier.
- *
- * <UL>
- * <LI>Working with different types.
- * <LI>Supplying default values
- * <LI>Dealing with null return types.
- * <LI>Working with row/column and column maps
- * </UL>
- *
- * <p>
- * This layer was intentionally loosely coupled with the basic API. This allows other convenience
- * layers for Fluo to build directly on the basic API w/o having to consider the particulars of this
- * layer. Also its expected that integration with other languages may only use the basic API.
- * </p>
- *
- * <h3>Using</h3>
- *
- * <p>
- * A TypeLayer is created with a certain encoder that is used for converting from bytes to
- * primitives and visa versa. In order to ensure that all of your code uses the same encoder, its
- * probably best to centralize the choice of an encoder within your project. There are many ways do
- * to this, below is an example of one way to centralize and use.
- * </p>
- *
- * <pre>
- * <code>
- *
- * public class MyTypeLayer extends TypeLayer {
- * public MyTypeLayer() {
- * super(new MyEncoder());
- * }
- * }
- *
- * public class MyObserver extends TypedObserver {
- * MyObserver(){
- * super(new MyTypeLayer());
- * }
- *
- * public abstract void process(TypedTransaction tx, Bytes row, Column col){
- * //do something w/ typed transaction
- * }
- * }
- *
- * public class MyUtil {
- * //A little util to print out some stuff
- * public void printStuff(Snapshot snap, byte[] row){
- * TypedSnapshot tsnap = new MytTypeLayer().wrap(snap);
- *
- * System.out.println(tsnap.get().row(row).fam("b90000").qual(137).toString("NP"));
- * }
- * }
- * </code>
- * </pre>
- *
- * <h3>Working with different types</h3>
- *
- * <p>
- * The following example code shows using the basic fluo API with different types.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void process(Transaction tx, byte[] row, byte[] cf, int cq, long val){
- * tx.set(Bytes.of(row), new Column(Bytes.of(cf), Bytes.of(Integer.toString(cq))),
- * Bytes.of(Long.toString(val));
- * }
- * </code>
- * </pre>
- *
- * <p>
- * Alternatively, the same thing can be written using a {@link TypedTransactionBase} in the
- * following way. Because row(), fam(), qual(), and set() each take many different types, this
- * enables many different permutations that would not be achievable with overloading.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void process(TypedTransaction tx, byte[] r, byte[] cf, int cq, long v){
- * tx.mutate().row(r).fam(cf).qual(cq).set(v);
- * }
- * </code>
- * </pre>
- *
- * <h3>Default values</h3>
- *
- * <p>
- * The following example code shows using the basic fluo API to read a value and default to zero if
- * it does not exist.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void add(Transaction tx, byte[] row, Column col, long amount){
- *
- * long balance = 0;
- * Bytes bval = tx.get(Bytes.of(row), col);
- * if(bval != null)
- * balance = Long.parseLong(bval.toString());
- *
- * balance += amount;
- *
- * tx.set(Bytes.of(row), col, Bytes.of(Long.toString(amount)));
- *
- * }
- * </code>
- * </pre>
- *
- * <p>
- * Alternatively, the same thing can be written using a {@link TypedTransactionBase} in the
- * following way. This code avoids the null check by supplying a default value of zero.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void add(TypedTransaction tx, byte[] r, Column c, long amount){
- * long balance = tx.get().row(r).col(c).toLong(0);
- * balance += amount;
- * tx.mutate().row(r).col(c).set(balance);
- * }
- * </code>
- * </pre>
- *
- * <p>
- * For this particular case, shorter code can be written by using the increment method.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void add(TypedTransaction tx, byte[] r, Column c, long amount){
- * tx.mutate().row(r).col(c).increment(amount);
- * }
- * </code>
- * </pre>
- *
- * <h3>Null return types</h3>
- *
- * <p>
- * When using the basic API, you must ensure the return type is not null before converting a string
- * or long.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void process(Transaction tx, byte[] row, Column col, long amount) {
- * Bytes val = tx.get(Bytes.of(row), col);
- * if(val == null)
- * return;
- * long balance = Long.parseLong(val.toString());
- * }
- * </code>
- * </pre>
- *
- * <p>
- * With {@link TypedTransactionBase} if no default value is supplied, then the null is passed
- * through.
- * </p>
- *
- * <pre>
- * <code>
- *
- * void process(TypedTransaction tx, byte[] r, Column c, long amount){
- * Long balance = tx.get().row(r).col(c).toLong();
- * if(balance == null)
- * return;
- * }
- * </code>
- * </pre>
- *
- * <h3>Defaulted maps</h3>
- *
- * <p>
- * The operations that return maps, return defaulted maps which make it easy to specify defaults and
- * avoid null.
- * </p>
- *
- * <pre>
- * {@code
- * // pretend this method has curly braces. javadoc has issues with less than.
- *
- * void process(TypedTransaction tx, byte[] r, Column c1, Column c2, Column c3, long amount)
- *
- * Map<Column, Value> columns = tx.get().row(r).columns(c1,c2,c3);
- *
- * // If c1 does not exist in map, a Value that wraps null will be returned.
- * // When c1 does not exist val1 will be set to null and no NPE will be thrown.
- * String val1 = columns.get(c1).toString();
- *
- * // If c2 does not exist in map, then val2 will be set to empty string.
- * String val2 = columns.get(c2).toString("");
- *
- * // If c3 does not exist in map, then val9 will be set to 9.
- * Long val3 = columns.get(c3).toLong(9);
- * }
- * </pre>
- *
- * <p>
- * This also applies to getting sets of rows.
- * </p>
- *
- * <pre>
- * {@code
- * // pretend this method has curly braces. javadoc has issues with less than.
- *
- * void process(TypedTransaction tx, List<String> rows, Column c1, Column c2, Column c3,
- * long amount)
- *
- * Map<String,Map<Column,Value>> rowCols =
- * tx.get().rowsString(rows).columns(c1,c2,c3).toStringMap();
- *
- * // this will set val1 to null if row does not exist in map and/or column does not
- * // exist in child map
- * String val1 = rowCols.get("row1").get(c1).toString();
- * }
- * </pre>
- *
- * @since 1.0.0
- */
-public class TypeLayer {
-
- private Encoder encoder;
-
- static class Data {
- Bytes row;
- Bytes family;
- Bytes qual;
- Bytes vis;
-
- Column getCol() {
- if (qual == null) {
- return new Column(family);
- } else if (vis == null) {
- return new Column(family, qual);
- } else {
- return new Column(family, qual, vis);
- }
- }
- }
-
- /**
- * @since 1.0.0
- */
- public abstract class RowMethods<R> {
-
- abstract R create(Data data);
-
- public R row(String row) {
- return row(encoder.encode(row));
- }
-
- public R row(int row) {
- return row(encoder.encode(row));
- }
-
- public R row(long row) {
- return row(encoder.encode(row));
- }
-
- public R row(byte[] row) {
- return row(Bytes.of(row));
- }
-
- public R row(ByteBuffer row) {
- return row(Bytes.of(row));
- }
-
- public R row(Bytes row) {
- Data data = new Data();
- data.row = row;
- R result = create(data);
- return result;
- }
- }
-
- /**
- * @since 1.0.0
- */
- public abstract class SimpleFamilyMethods<R1> {
-
- Data data;
-
- SimpleFamilyMethods(Data data) {
- this.data = data;
- }
-
- abstract R1 create1(Data data);
-
- public R1 fam(String family) {
- return fam(encoder.encode(family));
- }
-
- public R1 fam(int family) {
- return fam(encoder.encode(family));
- }
-
- public R1 fam(long family) {
- return fam(encoder.encode(family));
- }
-
- public R1 fam(byte[] family) {
- return fam(Bytes.of(family));
- }
-
- public R1 fam(ByteBuffer family) {
- return fam(Bytes.of(family));
- }
-
- public R1 fam(Bytes family) {
- data.family = family;
- return create1(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public abstract class FamilyMethods<R1, R2> extends SimpleFamilyMethods<R1> {
-
- FamilyMethods(Data data) {
- super(data);
- }
-
- abstract R2 create2(Data data);
-
- public R2 col(Column col) {
- data.family = col.getFamily();
- data.qual = col.getQualifier();
- data.vis = col.getVisibility();
- return create2(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public abstract class QualifierMethods<R> {
-
- private Data data;
-
- QualifierMethods(Data data) {
- this.data = data;
- }
-
- abstract R create(Data data);
-
- public R qual(String qualifier) {
- return qual(encoder.encode(qualifier));
- }
-
- public R qual(int qualifier) {
- return qual(encoder.encode(qualifier));
- }
-
- public R qual(long qualifier) {
- return qual(encoder.encode(qualifier));
- }
-
- public R qual(byte[] qualifier) {
- return qual(Bytes.of(qualifier));
- }
-
- public R qual(ByteBuffer qualifier) {
- return qual(Bytes.of(qualifier));
- }
-
- public R qual(Bytes qualifier) {
- data.qual = qualifier;
- return create(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public static class VisibilityMethods {
-
- private Data data;
-
- VisibilityMethods(Data data) {
- this.data = data;
- }
-
- public Column vis() {
- return new Column(data.family, data.qual);
- }
-
- public Column vis(String cv) {
- return vis(Bytes.of(cv));
- }
-
- public Column vis(Bytes cv) {
- return new Column(data.family, data.qual, cv);
- }
-
- public Column vis(ByteBuffer cv) {
- return vis(Bytes.of(cv));
- }
-
- public Column vis(byte[] cv) {
- return vis(Bytes.of(cv));
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class CQB extends QualifierMethods<VisibilityMethods> {
- CQB(Data data) {
- super(data);
- }
-
- @Override
- VisibilityMethods create(Data data) {
- return new VisibilityMethods(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class CFB extends SimpleFamilyMethods<CQB> {
- CFB() {
- super(new Data());
- }
-
- @Override
- CQB create1(Data data) {
- return new CQB(data);
- }
- }
-
- public TypeLayer(Encoder encoder) {
- this.encoder = encoder;
- }
-
- /**
- * Initiates the chain of calls needed to build a column.
- *
- * @return a column builder
- */
- public CFB bc() {
- return new CFB();
- }
-
- public TypedSnapshot wrap(Snapshot snap) {
- return new TypedSnapshot(snap, encoder, this);
- }
-
- public TypedTransactionBase wrap(TransactionBase tx) {
- return new TypedTransactionBase(tx, encoder, this);
- }
-
- public TypedTransaction wrap(Transaction tx) {
- return new TypedTransaction(tx, encoder, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedLoader.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedLoader.java
deleted file mode 100644
index be5625c..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedLoader.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.client.Loader;
-import org.apache.fluo.api.client.TransactionBase;
-
-/**
- * A {@link Loader} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public abstract class TypedLoader implements Loader {
-
- private final TypeLayer tl;
-
- public TypedLoader() {
- tl = new TypeLayer(new StringEncoder());
- }
-
- public TypedLoader(TypeLayer tl) {
- this.tl = tl;
- }
-
- @Override
- public void load(TransactionBase tx, Context context) throws Exception {
- load(tl.wrap(tx), context);
- }
-
- public abstract void load(TypedTransactionBase tx, Context context) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedObserver.java
deleted file mode 100644
index 799ed50..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedObserver.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.observer.AbstractObserver;
-
-/**
- * An {@link AbstractObserver} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public abstract class TypedObserver extends AbstractObserver {
-
- private final TypeLayer tl;
-
- public TypedObserver() {
- tl = new TypeLayer(new StringEncoder());
- }
-
- public TypedObserver(TypeLayer tl) {
- this.tl = tl;
- }
-
- @Override
- public void process(TransactionBase tx, Bytes row, Column col) {
- process(tl.wrap(tx), row, col);
- }
-
- public abstract void process(TypedTransactionBase tx, Bytes row, Column col);
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshot.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshot.java
deleted file mode 100644
index 033d4de..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshot.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import org.apache.fluo.api.client.Snapshot;
-
-/**
- * A {@link Snapshot} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public class TypedSnapshot extends TypedSnapshotBase implements Snapshot {
-
- private final Snapshot closeSnapshot;
-
- TypedSnapshot(Snapshot snapshot, Encoder encoder, TypeLayer tl) {
- super(snapshot, encoder, tl);
- closeSnapshot = snapshot;
- }
-
- @Override
- public void close() {
- closeSnapshot.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshotBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshotBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshotBase.java
deleted file mode 100644
index 30b807f..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedSnapshotBase.java
+++ /dev/null
@@ -1,555 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import com.google.common.collect.Maps;
-import org.apache.commons.collections.map.DefaultedMap;
-import org.apache.fluo.api.client.SnapshotBase;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumn;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.recipes.types.TypeLayer.Data;
-import org.apache.fluo.recipes.types.TypeLayer.FamilyMethods;
-import org.apache.fluo.recipes.types.TypeLayer.QualifierMethods;
-import org.apache.fluo.recipes.types.TypeLayer.RowMethods;
-
-// TODO need to refactor column to use Encoder
-
-/**
- * A {@link SnapshotBase} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public class TypedSnapshotBase implements SnapshotBase {
-
- private SnapshotBase snapshot;
- private Encoder encoder;
- private TypeLayer tl;
-
- /**
- * @since 1.0.0
- */
- public class VisibilityMethods extends Value {
-
- VisibilityMethods(Data data) {
- super(data);
- }
-
- public Value vis(Bytes cv) {
- data.vis = cv;
- return new Value(data);
- }
-
- public Value vis(byte[] cv) {
- data.vis = Bytes.of(cv);
- return new Value(data);
- }
-
- public Value vis(ByteBuffer bb) {
- data.vis = Bytes.of(bb);
- return new Value(data);
- }
-
- public Value vis(String cv) {
- data.vis = Bytes.of(cv);
- return new Value(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class Value {
- private Bytes bytes;
- private boolean gotBytes = false;
- Data data;
-
- public Bytes getBytes() {
- if (!gotBytes) {
- try {
- bytes = snapshot.get(data.row, data.getCol());
- gotBytes = true;
- } catch (Exception e) {
- if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- }
- throw new RuntimeException(e);
- }
- }
-
- return bytes;
- }
-
- private Value(Bytes bytes) {
- this.bytes = bytes;
- this.gotBytes = true;
- }
-
- private Value(Data data) {
- this.data = data;
- this.gotBytes = false;
- }
-
- public Integer toInteger() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeInteger(getBytes());
- }
-
- public int toInteger(int defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeInteger(getBytes());
- }
-
- public Long toLong() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeLong(getBytes());
- }
-
- public long toLong(long defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeLong(getBytes());
- }
-
- @Override
- public String toString() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeString(getBytes());
- }
-
- public String toString(String defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeString(getBytes());
- }
-
- public Float toFloat() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeFloat(getBytes());
- }
-
- public float toFloat(float defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeFloat(getBytes());
- }
-
- public Double toDouble() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeDouble(getBytes());
- }
-
- public double toDouble(double defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeDouble(getBytes());
- }
-
- public Boolean toBoolean() {
- if (getBytes() == null) {
- return null;
- }
- return encoder.decodeBoolean(getBytes());
- }
-
- public boolean toBoolean(boolean defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return encoder.decodeBoolean(getBytes());
- }
-
- public byte[] toBytes() {
- if (getBytes() == null) {
- return null;
- }
- return getBytes().toArray();
- }
-
- public byte[] toBytes(byte[] defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return getBytes().toArray();
- }
-
- public ByteBuffer toByteBuffer() {
- if (getBytes() == null) {
- return null;
- }
- return ByteBuffer.wrap(getBytes().toArray());
- }
-
- public ByteBuffer toByteBuffer(ByteBuffer defaultValue) {
- if (getBytes() == null) {
- return defaultValue;
- }
- return toByteBuffer();
- }
-
- @Override
- public int hashCode() {
- if (getBytes() == null) {
- return 0;
- }
-
- return getBytes().hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof Value) {
- Value ov = (Value) o;
- if (getBytes() == null) {
- return ov.getBytes() == null;
- } else {
- return getBytes().equals(ov.getBytes());
- }
- }
-
- return false;
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class ValueQualifierBuilder extends QualifierMethods<VisibilityMethods> {
-
- ValueQualifierBuilder(Data data) {
- tl.super(data);
- }
-
- @Override
- VisibilityMethods create(Data data) {
- return new VisibilityMethods(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class ValueFamilyMethods extends FamilyMethods<ValueQualifierBuilder, Value> {
-
- ValueFamilyMethods(Data data) {
- tl.super(data);
- }
-
- @Override
- ValueQualifierBuilder create1(Data data) {
- return new ValueQualifierBuilder(data);
- }
-
- @Override
- Value create2(Data data) {
- return new Value(data);
- }
-
- public Map<Column, Value> columns(Set<Column> columns) {
- try {
- return wrap(snapshot.get(data.row, columns));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- public Map<Column, Value> columns(Column... columns) {
- try {
- return wrap(snapshot.get(data.row, new HashSet<>(Arrays.asList(columns))));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class MapConverter {
- private Collection<Bytes> rows;
- private Set<Column> columns;
-
- public MapConverter(Collection<Bytes> rows, Set<Column> columns) {
- this.rows = rows;
- this.columns = columns;
- }
-
- private Map<Bytes, Map<Column, Bytes>> getInput() {
- try {
- return snapshot.get(rows, columns);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private Map wrap2(Map m) {
- return Collections.unmodifiableMap(DefaultedMap.decorate(m, new DefaultedMap(new Value(
- (Bytes) null))));
- }
-
- @SuppressWarnings("unchecked")
- public Map<String, Map<Column, Value>> toStringMap() {
- Map<Bytes, Map<Column, Bytes>> in = getInput();
- Map<String, Map<Column, Value>> out = new HashMap<>();
-
- for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
- out.put(encoder.decodeString(rowEntry.getKey()), wrap(rowEntry.getValue()));
- }
-
- return wrap2(out);
- }
-
- @SuppressWarnings("unchecked")
- public Map<Long, Map<Column, Value>> toLongMap() {
- Map<Bytes, Map<Column, Bytes>> in = getInput();
- Map<Long, Map<Column, Value>> out = new HashMap<>();
-
- for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
- out.put(encoder.decodeLong(rowEntry.getKey()), wrap(rowEntry.getValue()));
- }
-
- return wrap2(out);
- }
-
- @SuppressWarnings("unchecked")
- public Map<Integer, Map<Column, Value>> toIntegerMap() {
- Map<Bytes, Map<Column, Bytes>> in = getInput();
- Map<Integer, Map<Column, Value>> out = new HashMap<>();
-
- for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
- out.put(encoder.decodeInteger(rowEntry.getKey()), wrap(rowEntry.getValue()));
- }
-
- return wrap2(out);
- }
-
- @SuppressWarnings("unchecked")
- public Map<Bytes, Map<Column, Value>> toBytesMap() {
- Map<Bytes, Map<Column, Bytes>> in = getInput();
- Map<Bytes, Map<Column, Value>> out = new HashMap<>();
-
- for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
- out.put(rowEntry.getKey(), wrap(rowEntry.getValue()));
- }
-
- return wrap2(out);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class ColumnsMethods {
- private Collection<Bytes> rows;
-
- public ColumnsMethods(Collection<Bytes> rows) {
- this.rows = rows;
- }
-
- public MapConverter columns(Set<Column> columns) {
- return new MapConverter(rows, columns);
- }
-
- public MapConverter columns(Column... columns) {
- return columns(new HashSet<>(Arrays.asList(columns)));
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class ValueRowMethods extends RowMethods<ValueFamilyMethods> {
-
- ValueRowMethods() {
- tl.super();
- }
-
- @Override
- ValueFamilyMethods create(Data data) {
- return new ValueFamilyMethods(data);
- }
-
- public ColumnsMethods rows(Collection<Bytes> rows) {
- return new ColumnsMethods(rows);
- }
-
- public ColumnsMethods rows(Bytes... rows) {
- return new ColumnsMethods(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsString(String... rows) {
- return rowsString(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsString(Collection<String> rows) {
- ArrayList<Bytes> conv = new ArrayList<>();
- for (String row : rows) {
- conv.add(encoder.encode(row));
- }
-
- return rows(conv);
- }
-
- public ColumnsMethods rowsLong(Long... rows) {
- return rowsLong(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsLong(Collection<Long> rows) {
- ArrayList<Bytes> conv = new ArrayList<>();
- for (Long row : rows) {
- conv.add(encoder.encode(row));
- }
-
- return rows(conv);
- }
-
- public ColumnsMethods rowsInteger(Integer... rows) {
- return rowsInteger(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsInteger(Collection<Integer> rows) {
- ArrayList<Bytes> conv = new ArrayList<>();
- for (Integer row : rows) {
- conv.add(encoder.encode(row));
- }
-
- return rows(conv);
- }
-
- public ColumnsMethods rowsBytes(byte[]... rows) {
- return rowsBytes(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsBytes(Collection<byte[]> rows) {
- ArrayList<Bytes> conv = new ArrayList<>();
- for (byte[] row : rows) {
- conv.add(Bytes.of(row));
- }
-
- return rows(conv);
- }
-
- public ColumnsMethods rowsByteBuffers(ByteBuffer... rows) {
- return rowsByteBuffers(Arrays.asList(rows));
- }
-
- public ColumnsMethods rowsByteBuffers(Collection<ByteBuffer> rows) {
- ArrayList<Bytes> conv = new ArrayList<>();
- for (ByteBuffer row : rows) {
- conv.add(Bytes.of(row));
- }
-
- return rows(conv);
- }
-
- }
-
- TypedSnapshotBase(SnapshotBase snapshot, Encoder encoder, TypeLayer tl) {
- this.snapshot = snapshot;
- this.encoder = encoder;
- this.tl = tl;
- }
-
- @Override
- public Bytes get(Bytes row, Column column) {
- return snapshot.get(row, column);
- }
-
- @Override
- public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
- return snapshot.get(row, columns);
- }
-
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
- return snapshot.get(rowColumns);
- }
-
- @Override
- public RowIterator get(ScannerConfiguration config) {
- return snapshot.get(config);
- }
-
- @Override
- public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
- return snapshot.get(rows, columns);
- }
-
- public ValueRowMethods get() {
- return new ValueRowMethods();
- }
-
- @SuppressWarnings({"unchecked"})
- private Map<Column, Value> wrap(Map<Column, Bytes> map) {
- Map<Column, Value> ret = Maps.transformValues(map, input -> new Value(input));
- return Collections.unmodifiableMap(DefaultedMap.decorate(ret, new Value((Bytes) null)));
- }
-
- @Override
- public long getStartTimestamp() {
- return snapshot.getStartTimestamp();
- }
-
- @Override
- public String gets(String row, Column column) {
- return snapshot.gets(row, column);
- }
-
- @Override
- public Map<Column, String> gets(String row, Set<Column> columns) {
- return snapshot.gets(row, columns);
- }
-
- @Override
- public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
- return snapshot.gets(rows, columns);
- }
-
- @Override
- public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
- return snapshot.gets(rowColumns);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransaction.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransaction.java
deleted file mode 100644
index 1e22cd4..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransaction.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.exceptions.CommitException;
-
-/**
- * A {@link Transaction} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public class TypedTransaction extends TypedTransactionBase implements Transaction {
-
- private final Transaction closeTx;
-
- @VisibleForTesting
- protected TypedTransaction(Transaction tx, Encoder encoder, TypeLayer tl) {
- super(tx, encoder, tl);
- closeTx = tx;
- }
-
- @Override
- public void commit() throws CommitException {
- closeTx.commit();
- }
-
- @Override
- public void close() {
- closeTx.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransactionBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransactionBase.java
deleted file mode 100644
index 7972c65..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/types/TypedTransactionBase.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.types;
-
-import java.nio.ByteBuffer;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.exceptions.AlreadySetException;
-import org.apache.fluo.recipes.types.TypeLayer.Data;
-import org.apache.fluo.recipes.types.TypeLayer.FamilyMethods;
-import org.apache.fluo.recipes.types.TypeLayer.QualifierMethods;
-import org.apache.fluo.recipes.types.TypeLayer.RowMethods;
-
-/**
- * A {@link TransactionBase} that uses a {@link TypeLayer}
- *
- * @since 1.0.0
- */
-public class TypedTransactionBase extends TypedSnapshotBase implements TransactionBase {
-
- private final TransactionBase tx;
- private final Encoder encoder;
- private final TypeLayer tl;
-
- /**
- * @since 1.0.0
- */
- public class Mutator {
-
- private boolean set = false;
- Data data;
-
- Mutator(Data data) {
- this.data = data;
- }
-
- void checkNotSet() {
- if (set) {
- throw new IllegalStateException("Already set value");
- }
- }
-
- public void set(Bytes bytes) throws AlreadySetException {
- checkNotSet();
- tx.set(data.row, data.getCol(), bytes);
- set = true;
- }
-
- public void set(String s) throws AlreadySetException {
- set(encoder.encode(s));
- }
-
- public void set(int i) throws AlreadySetException {
- set(encoder.encode(i));
- }
-
- public void set(long l) throws AlreadySetException {
- set(encoder.encode(l));
- }
-
- public void set(float f) throws AlreadySetException {
- set(encoder.encode(f));
- }
-
- public void set(double d) throws AlreadySetException {
- set(encoder.encode(d));
- }
-
- public void set(boolean b) throws AlreadySetException {
- set(encoder.encode(b));
- }
-
- public void set(byte[] ba) throws AlreadySetException {
- set(Bytes.of(ba));
- }
-
- public void set(ByteBuffer bb) throws AlreadySetException {
- set(Bytes.of(bb));
- }
-
- /**
- * Set an empty value
- */
- public void set() throws AlreadySetException {
- set(Bytes.EMPTY);
- }
-
- /**
- * Reads the current value of the row/column, adds i, sets the sum. If the row/column does not
- * have a current value, then it defaults to zero.
- *
- * @param i Integer increment amount
- * @throws AlreadySetException if value was previously set in transaction
- */
- public void increment(int i) throws AlreadySetException {
- checkNotSet();
- Bytes val = tx.get(data.row, data.getCol());
- int v = 0;
- if (val != null) {
- v = encoder.decodeInteger(val);
- }
- tx.set(data.row, data.getCol(), encoder.encode(v + i));
- }
-
- /**
- * Reads the current value of the row/column, adds l, sets the sum. If the row/column does not
- * have a current value, then it defaults to zero.
- *
- * @param l Long increment amount
- * @throws AlreadySetException if value was previously set in transaction
- */
- public void increment(long l) throws AlreadySetException {
- checkNotSet();
- Bytes val = tx.get(data.row, data.getCol());
- long v = 0;
- if (val != null) {
- v = encoder.decodeLong(val);
- }
- tx.set(data.row, data.getCol(), encoder.encode(v + l));
- }
-
- public void delete() throws AlreadySetException {
- checkNotSet();
- tx.delete(data.row, data.getCol());
- set = true;
- }
-
- public void weaklyNotify() {
- checkNotSet();
- tx.setWeakNotification(data.row, data.getCol());
- set = true;
- }
-
- }
-
- /**
- * @since 1.0.0
- */
- public class VisibilityMutator extends Mutator {
-
- VisibilityMutator(Data data) {
- super(data);
- }
-
- public Mutator vis(String cv) {
- checkNotSet();
- data.vis = Bytes.of(cv);
- return new Mutator(data);
- }
-
- public Mutator vis(Bytes cv) {
- checkNotSet();
- data.vis = cv;
- return new Mutator(data);
- }
-
- public Mutator vis(byte[] cv) {
- checkNotSet();
- data.vis = Bytes.of(cv);
- return new Mutator(data);
- }
-
- public Mutator vis(ByteBuffer cv) {
- checkNotSet();
- data.vis = Bytes.of(cv);
- return new Mutator(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class MutatorQualifierMethods extends QualifierMethods<VisibilityMutator> {
-
- MutatorQualifierMethods(Data data) {
- tl.super(data);
- }
-
- @Override
- VisibilityMutator create(Data data) {
- return new VisibilityMutator(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class MutatorFamilyMethods extends FamilyMethods<MutatorQualifierMethods, Mutator> {
-
- MutatorFamilyMethods(Data data) {
- tl.super(data);
- }
-
- @Override
- MutatorQualifierMethods create1(Data data) {
- return new MutatorQualifierMethods(data);
- }
-
- @Override
- Mutator create2(Data data) {
- return new Mutator(data);
- }
- }
-
- /**
- * @since 1.0.0
- */
- public class MutatorRowMethods extends RowMethods<MutatorFamilyMethods> {
-
- MutatorRowMethods() {
- tl.super();
- }
-
- @Override
- MutatorFamilyMethods create(Data data) {
- return new MutatorFamilyMethods(data);
- }
-
- }
-
- @VisibleForTesting
- protected TypedTransactionBase(TransactionBase tx, Encoder encoder, TypeLayer tl) {
- super(tx, encoder, tl);
- this.tx = tx;
- this.encoder = encoder;
- this.tl = tl;
- }
-
- public MutatorRowMethods mutate() {
- return new MutatorRowMethods();
- }
-
- @Override
- public void set(Bytes row, Column col, Bytes value) throws AlreadySetException {
- tx.set(row, col, value);
- }
-
- @Override
- public void set(String row, Column col, String value) throws AlreadySetException {
- tx.set(row, col, value);
- }
-
- @Override
- public void setWeakNotification(Bytes row, Column col) {
- tx.setWeakNotification(row, col);
- }
-
- @Override
- public void setWeakNotification(String row, Column col) {
- tx.setWeakNotification(row, col);
- }
-
- @Override
- public void delete(Bytes row, Column col) throws AlreadySetException {
- tx.delete(row, col);
- }
-
- @Override
- public void delete(String row, Column col) {
- tx.delete(row, col);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/common/TestGrouping.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/common/TestGrouping.java b/modules/core/src/test/java/org/apache/fluo/recipes/common/TestGrouping.java
deleted file mode 100644
index 8bf338c..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/common/TestGrouping.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.common;
-
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.export.ExportQueue;
-import org.apache.fluo.recipes.map.CollisionFreeMap;
-import org.apache.fluo.recipes.map.CollisionFreeMap.Options;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestGrouping {
- @Test
- public void testTabletGrouping() {
- FluoConfiguration conf = new FluoConfiguration();
-
- CollisionFreeMap.configure(conf, new Options("m1", "ct", "kt", "vt", 119));
- CollisionFreeMap.configure(conf, new Options("m2", "ct", "kt", "vt", 3));
-
- ExportQueue.configure(conf, new org.apache.fluo.recipes.export.ExportQueue.Options("eq1", "et",
- "kt", "vt", 7));
- ExportQueue.configure(conf, new org.apache.fluo.recipes.export.ExportQueue.Options("eq2", "et",
- "kt", "vt", 3));
-
- Pirtos pirtos = CollisionFreeMap.getTableOptimizations(conf.getAppConfiguration());
- pirtos.merge(ExportQueue.getTableOptimizations(conf.getAppConfiguration()));
-
- Pattern pattern = Pattern.compile(pirtos.getTabletGroupingRegex());
-
- Assert.assertEquals("m1:u:", group(pattern, "m1:u:f0c"));
- Assert.assertEquals("m1:d:", group(pattern, "m1:d:f0c"));
- Assert.assertEquals("m2:u:", group(pattern, "m2:u:abc"));
- Assert.assertEquals("m2:d:", group(pattern, "m2:d:590"));
- Assert.assertEquals("none", group(pattern, "m3:d:590"));
-
- Assert.assertEquals("eq1:", group(pattern, "eq1:f0c"));
- Assert.assertEquals("eq2:", group(pattern, "eq2:f0c"));
- Assert.assertEquals("none", group(pattern, "eq3:f0c"));
-
- // validate the assumptions this test is making
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1#")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2#")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq1:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("eq2:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:u:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m1:d:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:u:~")));
- Assert.assertTrue(pirtos.getSplits().contains(Bytes.of("m2:d:~")));
-
- Set<String> expectedGroups =
- ImmutableSet.of("m1:u:", "m1:d:", "m2:u:", "m2:d:", "eq1:", "eq2:");
-
- // ensure all splits group as expected
- for (Bytes split : pirtos.getSplits()) {
- String g = group(pattern, split.toString());
-
- if (expectedGroups.contains(g)) {
- Assert.assertTrue(split.toString().startsWith(g));
- } else {
- Assert.assertEquals("none", g);
- Assert.assertTrue(split.toString().equals("eq1#") || split.toString().equals("eq2#"));
- }
-
- }
-
- }
-
- private String group(Pattern pattern, String endRow) {
- Matcher m = pattern.matcher(endRow);
- if (m.matches() && m.groupCount() == 1) {
- return m.group(1);
- }
- return "none";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/test/java/org/apache/fluo/recipes/common/TransientRegistryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/fluo/recipes/common/TransientRegistryTest.java b/modules/core/src/test/java/org/apache/fluo/recipes/common/TransientRegistryTest.java
deleted file mode 100644
index b30261f..0000000
--- a/modules/core/src/test/java/org/apache/fluo/recipes/common/TransientRegistryTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.common;
-
-import java.util.HashSet;
-
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TransientRegistryTest {
- @Test
- public void testBasic() {
- FluoConfiguration fluoConfig = new FluoConfiguration();
-
- HashSet<RowRange> expected = new HashSet<>();
-
- TransientRegistry tr = new TransientRegistry(fluoConfig.getAppConfiguration());
-
- RowRange rr1 = new RowRange(Bytes.of("pr1:g"), Bytes.of("pr1:q"));
- tr.addTransientRange("foo", rr1);
- expected.add(rr1);
-
- tr = new TransientRegistry(fluoConfig.getAppConfiguration());
- Assert.assertEquals(expected, new HashSet<>(tr.getTransientRanges()));
-
- RowRange rr2 = new RowRange(Bytes.of("pr2:j"), Bytes.of("pr2:m"));
- tr.addTransientRange("bar", rr2);
- expected.add(rr2);
-
- tr = new TransientRegistry(fluoConfig.getAppConfiguration());
- Assert.assertEquals(expected, new HashSet<>(tr.getTransientRanges()));
- }
-}
[08/10] incubator-fluo-recipes git commit: Made SharedBatchWriter
package private
Posted by kt...@apache.org.
Made SharedBatchWriter package private
Project: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/commit/594997ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/tree/594997ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/diff/594997ab
Branch: refs/heads/master
Commit: 594997ab7afe4f5a0b57854e64f245b96816aaeb
Parents: beea3f9
Author: Mike Walch <mw...@gmail.com>
Authored: Fri Jul 15 11:50:40 2016 -0400
Committer: Mike Walch <mw...@gmail.com>
Committed: Fri Jul 15 11:50:40 2016 -0400
----------------------------------------------------------------------
.../recipes/accumulo/export/SharedBatchWriter.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/594997ab/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/SharedBatchWriter.java
----------------------------------------------------------------------
diff --git a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/SharedBatchWriter.java b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/SharedBatchWriter.java
index ddc7830..9d189e6 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/SharedBatchWriter.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/recipes/accumulo/export/SharedBatchWriter.java
@@ -36,14 +36,14 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Mutation;
-public class SharedBatchWriter {
+class SharedBatchWriter {
private static class Mutations {
List<Mutation> mutations;
CountDownLatch cdl = new CountDownLatch(1);
- public Mutations(Collection<Mutation> mutations) {
+ Mutations(Collection<Mutation> mutations) {
this.mutations = new ArrayList<>(mutations);
}
}
@@ -52,8 +52,8 @@ public class SharedBatchWriter {
private BatchWriter bw;
- public ExportTask(String instanceName, String zookeepers, String user, String password,
- String table) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ ExportTask(String instanceName, String zookeepers, String user, String password, String table)
+ throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
ZooKeeperInstance zki =
new ZooKeeperInstance(new ClientConfiguration().withInstance(instanceName).withZkHosts(
zookeepers));
@@ -107,7 +107,7 @@ public class SharedBatchWriter {
private static LinkedBlockingQueue<Mutations> exportQueue = null;
- public SharedBatchWriter(String instanceName, String zookeepers, String user, String password,
+ private SharedBatchWriter(String instanceName, String zookeepers, String user, String password,
String table) throws Exception {
// TODO: fix this write to static and remove findbugs max rank override in pom.xml
@@ -121,7 +121,7 @@ public class SharedBatchWriter {
private static Map<String, SharedBatchWriter> exporters = new HashMap<>();
- public static synchronized SharedBatchWriter getInstance(String instanceName, String zookeepers,
+ static synchronized SharedBatchWriter getInstance(String instanceName, String zookeepers,
String user, String password, String table) throws Exception {
String key =
@@ -137,7 +137,7 @@ public class SharedBatchWriter {
return ret;
}
- public void write(Collection<Mutation> mutations) {
+ void write(Collection<Mutation> mutations) {
Mutations work = new Mutations(mutations);
exportQueue.add(work);
try {
[05/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshotBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshotBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshotBase.java
new file mode 100644
index 0000000..7764e67
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshotBase.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.collections.map.DefaultedMap;
+import org.apache.fluo.api.client.SnapshotBase;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumn;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.recipes.core.types.TypeLayer.Data;
+import org.apache.fluo.recipes.core.types.TypeLayer.FamilyMethods;
+import org.apache.fluo.recipes.core.types.TypeLayer.QualifierMethods;
+import org.apache.fluo.recipes.core.types.TypeLayer.RowMethods;
+
+// TODO need to refactor column to use Encoder
+
+/**
+ * A {@link SnapshotBase} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public class TypedSnapshotBase implements SnapshotBase {
+
+ private SnapshotBase snapshot;
+ private Encoder encoder;
+ private TypeLayer tl;
+
+ /**
+ * @since 1.0.0
+ */
+ public class VisibilityMethods extends Value {
+
+ VisibilityMethods(Data data) {
+ super(data);
+ }
+
+ public Value vis(Bytes cv) {
+ data.vis = cv;
+ return new Value(data);
+ }
+
+ public Value vis(byte[] cv) {
+ data.vis = Bytes.of(cv);
+ return new Value(data);
+ }
+
+ public Value vis(ByteBuffer bb) {
+ data.vis = Bytes.of(bb);
+ return new Value(data);
+ }
+
+ public Value vis(String cv) {
+ data.vis = Bytes.of(cv);
+ return new Value(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class Value {
+ private Bytes bytes;
+ private boolean gotBytes = false;
+ Data data;
+
+ public Bytes getBytes() {
+ if (!gotBytes) {
+ try {
+ bytes = snapshot.get(data.row, data.getCol());
+ gotBytes = true;
+ } catch (Exception e) {
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ }
+ throw new RuntimeException(e);
+ }
+ }
+
+ return bytes;
+ }
+
+ private Value(Bytes bytes) {
+ this.bytes = bytes;
+ this.gotBytes = true;
+ }
+
+ private Value(Data data) {
+ this.data = data;
+ this.gotBytes = false;
+ }
+
+ public Integer toInteger() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeInteger(getBytes());
+ }
+
+ public int toInteger(int defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeInteger(getBytes());
+ }
+
+ public Long toLong() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeLong(getBytes());
+ }
+
+ public long toLong(long defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeLong(getBytes());
+ }
+
+ @Override
+ public String toString() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeString(getBytes());
+ }
+
+ public String toString(String defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeString(getBytes());
+ }
+
+ public Float toFloat() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeFloat(getBytes());
+ }
+
+ public float toFloat(float defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeFloat(getBytes());
+ }
+
+ public Double toDouble() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeDouble(getBytes());
+ }
+
+ public double toDouble(double defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeDouble(getBytes());
+ }
+
+ public Boolean toBoolean() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return encoder.decodeBoolean(getBytes());
+ }
+
+ public boolean toBoolean(boolean defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return encoder.decodeBoolean(getBytes());
+ }
+
+ public byte[] toBytes() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return getBytes().toArray();
+ }
+
+ public byte[] toBytes(byte[] defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return getBytes().toArray();
+ }
+
+ public ByteBuffer toByteBuffer() {
+ if (getBytes() == null) {
+ return null;
+ }
+ return ByteBuffer.wrap(getBytes().toArray());
+ }
+
+ public ByteBuffer toByteBuffer(ByteBuffer defaultValue) {
+ if (getBytes() == null) {
+ return defaultValue;
+ }
+ return toByteBuffer();
+ }
+
+ @Override
+ public int hashCode() {
+ if (getBytes() == null) {
+ return 0;
+ }
+
+ return getBytes().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Value) {
+ Value ov = (Value) o;
+ if (getBytes() == null) {
+ return ov.getBytes() == null;
+ } else {
+ return getBytes().equals(ov.getBytes());
+ }
+ }
+
+ return false;
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class ValueQualifierBuilder extends QualifierMethods<VisibilityMethods> {
+
+ ValueQualifierBuilder(Data data) {
+ tl.super(data);
+ }
+
+ @Override
+ VisibilityMethods create(Data data) {
+ return new VisibilityMethods(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class ValueFamilyMethods extends FamilyMethods<ValueQualifierBuilder, Value> {
+
+ ValueFamilyMethods(Data data) {
+ tl.super(data);
+ }
+
+ @Override
+ ValueQualifierBuilder create1(Data data) {
+ return new ValueQualifierBuilder(data);
+ }
+
+ @Override
+ Value create2(Data data) {
+ return new Value(data);
+ }
+
+ public Map<Column, Value> columns(Set<Column> columns) {
+ try {
+ return wrap(snapshot.get(data.row, columns));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Map<Column, Value> columns(Column... columns) {
+ try {
+ return wrap(snapshot.get(data.row, new HashSet<>(Arrays.asList(columns))));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class MapConverter {
+ private Collection<Bytes> rows;
+ private Set<Column> columns;
+
+ public MapConverter(Collection<Bytes> rows, Set<Column> columns) {
+ this.rows = rows;
+ this.columns = columns;
+ }
+
+ private Map<Bytes, Map<Column, Bytes>> getInput() {
+ try {
+ return snapshot.get(rows, columns);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private Map wrap2(Map m) {
+ return Collections.unmodifiableMap(DefaultedMap.decorate(m, new DefaultedMap(new Value(
+ (Bytes) null))));
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, Map<Column, Value>> toStringMap() {
+ Map<Bytes, Map<Column, Bytes>> in = getInput();
+ Map<String, Map<Column, Value>> out = new HashMap<>();
+
+ for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
+ out.put(encoder.decodeString(rowEntry.getKey()), wrap(rowEntry.getValue()));
+ }
+
+ return wrap2(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<Long, Map<Column, Value>> toLongMap() {
+ Map<Bytes, Map<Column, Bytes>> in = getInput();
+ Map<Long, Map<Column, Value>> out = new HashMap<>();
+
+ for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
+ out.put(encoder.decodeLong(rowEntry.getKey()), wrap(rowEntry.getValue()));
+ }
+
+ return wrap2(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<Integer, Map<Column, Value>> toIntegerMap() {
+ Map<Bytes, Map<Column, Bytes>> in = getInput();
+ Map<Integer, Map<Column, Value>> out = new HashMap<>();
+
+ for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
+ out.put(encoder.decodeInteger(rowEntry.getKey()), wrap(rowEntry.getValue()));
+ }
+
+ return wrap2(out);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<Bytes, Map<Column, Value>> toBytesMap() {
+ Map<Bytes, Map<Column, Bytes>> in = getInput();
+ Map<Bytes, Map<Column, Value>> out = new HashMap<>();
+
+ for (Entry<Bytes, Map<Column, Bytes>> rowEntry : in.entrySet()) {
+ out.put(rowEntry.getKey(), wrap(rowEntry.getValue()));
+ }
+
+ return wrap2(out);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class ColumnsMethods {
+ private Collection<Bytes> rows;
+
+ public ColumnsMethods(Collection<Bytes> rows) {
+ this.rows = rows;
+ }
+
+ public MapConverter columns(Set<Column> columns) {
+ return new MapConverter(rows, columns);
+ }
+
+ public MapConverter columns(Column... columns) {
+ return columns(new HashSet<>(Arrays.asList(columns)));
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class ValueRowMethods extends RowMethods<ValueFamilyMethods> {
+
+ ValueRowMethods() {
+ tl.super();
+ }
+
+ @Override
+ ValueFamilyMethods create(Data data) {
+ return new ValueFamilyMethods(data);
+ }
+
+ public ColumnsMethods rows(Collection<Bytes> rows) {
+ return new ColumnsMethods(rows);
+ }
+
+ public ColumnsMethods rows(Bytes... rows) {
+ return new ColumnsMethods(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsString(String... rows) {
+ return rowsString(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsString(Collection<String> rows) {
+ ArrayList<Bytes> conv = new ArrayList<>();
+ for (String row : rows) {
+ conv.add(encoder.encode(row));
+ }
+
+ return rows(conv);
+ }
+
+ public ColumnsMethods rowsLong(Long... rows) {
+ return rowsLong(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsLong(Collection<Long> rows) {
+ ArrayList<Bytes> conv = new ArrayList<>();
+ for (Long row : rows) {
+ conv.add(encoder.encode(row));
+ }
+
+ return rows(conv);
+ }
+
+ public ColumnsMethods rowsInteger(Integer... rows) {
+ return rowsInteger(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsInteger(Collection<Integer> rows) {
+ ArrayList<Bytes> conv = new ArrayList<>();
+ for (Integer row : rows) {
+ conv.add(encoder.encode(row));
+ }
+
+ return rows(conv);
+ }
+
+ public ColumnsMethods rowsBytes(byte[]... rows) {
+ return rowsBytes(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsBytes(Collection<byte[]> rows) {
+ ArrayList<Bytes> conv = new ArrayList<>();
+ for (byte[] row : rows) {
+ conv.add(Bytes.of(row));
+ }
+
+ return rows(conv);
+ }
+
+ public ColumnsMethods rowsByteBuffers(ByteBuffer... rows) {
+ return rowsByteBuffers(Arrays.asList(rows));
+ }
+
+ public ColumnsMethods rowsByteBuffers(Collection<ByteBuffer> rows) {
+ ArrayList<Bytes> conv = new ArrayList<>();
+ for (ByteBuffer row : rows) {
+ conv.add(Bytes.of(row));
+ }
+
+ return rows(conv);
+ }
+
+ }
+
+ TypedSnapshotBase(SnapshotBase snapshot, Encoder encoder, TypeLayer tl) {
+ this.snapshot = snapshot;
+ this.encoder = encoder;
+ this.tl = tl;
+ }
+
+ @Override
+ public Bytes get(Bytes row, Column column) {
+ return snapshot.get(row, column);
+ }
+
+ @Override
+ public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
+ return snapshot.get(row, columns);
+ }
+
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
+ return snapshot.get(rowColumns);
+ }
+
+ @Override
+ public RowIterator get(ScannerConfiguration config) {
+ return snapshot.get(config);
+ }
+
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
+ return snapshot.get(rows, columns);
+ }
+
+ public ValueRowMethods get() {
+ return new ValueRowMethods();
+ }
+
+ @SuppressWarnings({"unchecked"})
+ private Map<Column, Value> wrap(Map<Column, Bytes> map) {
+ Map<Column, Value> ret = Maps.transformValues(map, input -> new Value(input));
+ return Collections.unmodifiableMap(DefaultedMap.decorate(ret, new Value((Bytes) null)));
+ }
+
+ @Override
+ public long getStartTimestamp() {
+ return snapshot.getStartTimestamp();
+ }
+
+ @Override
+ public String gets(String row, Column column) {
+ return snapshot.gets(row, column);
+ }
+
+ @Override
+ public Map<Column, String> gets(String row, Set<Column> columns) {
+ return snapshot.gets(row, columns);
+ }
+
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
+ return snapshot.gets(rows, columns);
+ }
+
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
+ return snapshot.gets(rowColumns);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransaction.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransaction.java
new file mode 100644
index 0000000..17631e0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransaction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.exceptions.CommitException;
+
+/**
+ * A {@link Transaction} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public class TypedTransaction extends TypedTransactionBase implements Transaction {
+
+ private final Transaction closeTx;
+
+ @VisibleForTesting
+ protected TypedTransaction(Transaction tx, Encoder encoder, TypeLayer tl) {
+ super(tx, encoder, tl);
+ closeTx = tx;
+ }
+
+ @Override
+ public void commit() throws CommitException {
+ closeTx.commit();
+ }
+
+ @Override
+ public void close() {
+ closeTx.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransactionBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransactionBase.java
new file mode 100644
index 0000000..69ec694
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedTransactionBase.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.nio.ByteBuffer;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.exceptions.AlreadySetException;
+import org.apache.fluo.recipes.core.types.TypeLayer.Data;
+import org.apache.fluo.recipes.core.types.TypeLayer.FamilyMethods;
+import org.apache.fluo.recipes.core.types.TypeLayer.QualifierMethods;
+import org.apache.fluo.recipes.core.types.TypeLayer.RowMethods;
+
+/**
+ * A {@link TransactionBase} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public class TypedTransactionBase extends TypedSnapshotBase implements TransactionBase {
+
+ private final TransactionBase tx;
+ private final Encoder encoder;
+ private final TypeLayer tl;
+
+ /**
+ * @since 1.0.0
+ */
+ public class Mutator {
+
+ private boolean set = false;
+ Data data;
+
+ Mutator(Data data) {
+ this.data = data;
+ }
+
+ void checkNotSet() {
+ if (set) {
+ throw new IllegalStateException("Already set value");
+ }
+ }
+
+ public void set(Bytes bytes) throws AlreadySetException {
+ checkNotSet();
+ tx.set(data.row, data.getCol(), bytes);
+ set = true;
+ }
+
+ public void set(String s) throws AlreadySetException {
+ set(encoder.encode(s));
+ }
+
+ public void set(int i) throws AlreadySetException {
+ set(encoder.encode(i));
+ }
+
+ public void set(long l) throws AlreadySetException {
+ set(encoder.encode(l));
+ }
+
+ public void set(float f) throws AlreadySetException {
+ set(encoder.encode(f));
+ }
+
+ public void set(double d) throws AlreadySetException {
+ set(encoder.encode(d));
+ }
+
+ public void set(boolean b) throws AlreadySetException {
+ set(encoder.encode(b));
+ }
+
+ public void set(byte[] ba) throws AlreadySetException {
+ set(Bytes.of(ba));
+ }
+
+ public void set(ByteBuffer bb) throws AlreadySetException {
+ set(Bytes.of(bb));
+ }
+
+ /**
+ * Set an empty value
+ */
+ public void set() throws AlreadySetException {
+ set(Bytes.EMPTY);
+ }
+
+ /**
+ * Reads the current value of the row/column, adds i, sets the sum. If the row/column does not
+ * have a current value, then it defaults to zero.
+ *
+ * @param i Integer increment amount
+ * @throws AlreadySetException if value was previously set in transaction
+ */
+ public void increment(int i) throws AlreadySetException {
+ checkNotSet();
+ Bytes val = tx.get(data.row, data.getCol());
+ int v = 0;
+ if (val != null) {
+ v = encoder.decodeInteger(val);
+ }
+ tx.set(data.row, data.getCol(), encoder.encode(v + i));
+ }
+
+ /**
+ * Reads the current value of the row/column, adds l, sets the sum. If the row/column does not
+ * have a current value, then it defaults to zero.
+ *
+ * @param l Long increment amount
+ * @throws AlreadySetException if value was previously set in transaction
+ */
+ public void increment(long l) throws AlreadySetException {
+ checkNotSet();
+ Bytes val = tx.get(data.row, data.getCol());
+ long v = 0;
+ if (val != null) {
+ v = encoder.decodeLong(val);
+ }
+ tx.set(data.row, data.getCol(), encoder.encode(v + l));
+ }
+
+ public void delete() throws AlreadySetException {
+ checkNotSet();
+ tx.delete(data.row, data.getCol());
+ set = true;
+ }
+
+ public void weaklyNotify() {
+ checkNotSet();
+ tx.setWeakNotification(data.row, data.getCol());
+ set = true;
+ }
+
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class VisibilityMutator extends Mutator {
+
+ VisibilityMutator(Data data) {
+ super(data);
+ }
+
+ public Mutator vis(String cv) {
+ checkNotSet();
+ data.vis = Bytes.of(cv);
+ return new Mutator(data);
+ }
+
+ public Mutator vis(Bytes cv) {
+ checkNotSet();
+ data.vis = cv;
+ return new Mutator(data);
+ }
+
+ public Mutator vis(byte[] cv) {
+ checkNotSet();
+ data.vis = Bytes.of(cv);
+ return new Mutator(data);
+ }
+
+ public Mutator vis(ByteBuffer cv) {
+ checkNotSet();
+ data.vis = Bytes.of(cv);
+ return new Mutator(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class MutatorQualifierMethods extends QualifierMethods<VisibilityMutator> {
+
+ MutatorQualifierMethods(Data data) {
+ tl.super(data);
+ }
+
+ @Override
+ VisibilityMutator create(Data data) {
+ return new VisibilityMutator(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class MutatorFamilyMethods extends FamilyMethods<MutatorQualifierMethods, Mutator> {
+
+ MutatorFamilyMethods(Data data) {
+ tl.super(data);
+ }
+
+ @Override
+ MutatorQualifierMethods create1(Data data) {
+ return new MutatorQualifierMethods(data);
+ }
+
+ @Override
+ Mutator create2(Data data) {
+ return new Mutator(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class MutatorRowMethods extends RowMethods<MutatorFamilyMethods> {
+
+ MutatorRowMethods() {
+ tl.super();
+ }
+
+ @Override
+ MutatorFamilyMethods create(Data data) {
+ return new MutatorFamilyMethods(data);
+ }
+
+ }
+
+ @VisibleForTesting
+ protected TypedTransactionBase(TransactionBase tx, Encoder encoder, TypeLayer tl) {
+ super(tx, encoder, tl);
+ this.tx = tx;
+ this.encoder = encoder;
+ this.tl = tl;
+ }
+
+ public MutatorRowMethods mutate() {
+ return new MutatorRowMethods();
+ }
+
+ @Override
+ public void set(Bytes row, Column col, Bytes value) throws AlreadySetException {
+ tx.set(row, col, value);
+ }
+
+ @Override
+ public void set(String row, Column col, String value) throws AlreadySetException {
+ tx.set(row, col, value);
+ }
+
+ @Override
+ public void setWeakNotification(Bytes row, Column col) {
+ tx.setWeakNotification(row, col);
+ }
+
+ @Override
+ public void setWeakNotification(String row, Column col) {
+ tx.setWeakNotification(row, col);
+ }
+
+ @Override
+ public void delete(Bytes row, Column col) throws AlreadySetException {
+ tx.delete(row, col);
+ }
+
+ @Override
+ public void delete(String row, Column col) {
+ tx.delete(row, col);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/data/RowHasher.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/data/RowHasher.java b/modules/core/src/main/java/org/apache/fluo/recipes/data/RowHasher.java
deleted file mode 100644
index 2501fa1..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/data/RowHasher.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.data;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.hash.Hashing;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.BytesBuilder;
-import org.apache.fluo.recipes.common.Pirtos;
-
-/**
- * This recipe provides code to help add a hash of the row as a prefix of the row. Using this recipe
- * rows are structured like the following.
- *
- * <p>
- * {@code <prefix>:<fixed len row hash>:<user row>}
- *
- * <p>
- * The recipe also provides code the help generate split points and configure balancing of the
- * prefix.
- *
- * <p>
- * The project documentation has more information.
- */
-public class RowHasher {
-
- private static final int HASH_LEN = 4;
-
- public Pirtos getTableOptimizations(int numTablets) {
-
- List<Bytes> splits = new ArrayList<>(numTablets - 1);
-
- int numSplits = numTablets - 1;
- int distance = (((int) Math.pow(Character.MAX_RADIX, HASH_LEN) - 1) / numTablets) + 1;
- int split = distance;
- for (int i = 0; i < numSplits; i++) {
- splits.add(Bytes.of(prefix
- + Strings.padStart(Integer.toString(split, Character.MAX_RADIX), HASH_LEN, '0')));
- split += distance;
- }
-
- splits.add(Bytes.of(prefix + "~"));
-
-
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
- pirtos.setTabletGroupingRegex(Pattern.quote(prefix.toString()));
-
- return pirtos;
- }
-
-
- private Bytes prefix;
-
- public RowHasher(String prefix) {
- this.prefix = Bytes.of(prefix + ":");
- }
-
- /**
- * @return Returns input with prefix and hash of input prepended.
- */
- public Bytes addHash(String row) {
- return addHash(Bytes.of(row));
- }
-
- /**
- * @return Returns input with prefix and hash of input prepended.
- */
- public Bytes addHash(Bytes row) {
- BytesBuilder builder = Bytes.newBuilder(prefix.length() + 5 + row.length());
- builder.append(prefix);
- builder.append(genHash(row));
- builder.append(":");
- builder.append(row);
- return builder.toBytes();
- }
-
- private boolean hasHash(Bytes row) {
- for (int i = prefix.length(); i < prefix.length() + HASH_LEN; i++) {
- byte b = row.byteAt(i);
- boolean isAlphaNum = (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9');
- if (!isAlphaNum) {
- return false;
- }
- }
-
- if (row.byteAt(prefix.length() - 1) != ':' || row.byteAt(prefix.length() + HASH_LEN) != ':') {
- return false;
- }
-
- return true;
- }
-
- /**
- * @return Returns input with prefix and hash stripped from beginning.
- */
- public Bytes removeHash(Bytes row) {
- Preconditions.checkArgument(row.length() >= prefix.length() + 5,
- "Row is shorter than expected " + row);
- Preconditions.checkArgument(row.subSequence(0, prefix.length()).equals(prefix),
- "Row does not have expected prefix " + row);
- Preconditions.checkArgument(hasHash(row), "Row does not have expected hash " + row);
- return row.subSequence(prefix.length() + 5, row.length());
- }
-
- private static String genHash(Bytes row) {
- int hash = Hashing.murmur3_32().hashBytes(row.toArray()).asInt();
- hash = hash & 0x7fffffff;
- // base 36 gives a lot more bins in 4 bytes than hex, but it is still human readable which is
- // nice for debugging.
- String hashString =
- Strings.padStart(Integer.toString(hash, Character.MAX_RADIX), HASH_LEN, '0');
- hashString = hashString.substring(hashString.length() - HASH_LEN);
-
- return hashString;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/Export.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/Export.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/Export.java
deleted file mode 100644
index c477ab1..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/Export.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Objects;
-
-public class Export<K, V> {
- private final K key;
- private final V value;
-
- public Export(K key, V val) {
- Objects.requireNonNull(key);
- Objects.requireNonNull(val);
- this.key = key;
- this.value = val;
- }
-
- public K getKey() {
- return key;
- }
-
- public V getValue() {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportBucket.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportBucket.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportBucket.java
deleted file mode 100644
index fa9bb45..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportBucket.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import com.google.common.base.Preconditions;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumn;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.recipes.impl.BucketUtil;
-import org.apache.fluo.recipes.types.StringEncoder;
-import org.apache.fluo.recipes.types.TypeLayer;
-import org.apache.fluo.recipes.types.TypedTransactionBase;
-
-/**
- * This class encapsulates a buckets serialization code.
- */
-class ExportBucket {
- private static final String NOTIFICATION_CF = "fluoRecipes";
- private static final String NOTIFICATION_CQ_PREFIX = "eq:";
- private static final Column EXPORT_COL = new Column("e", "v");
- private static final Column NEXT_COL = new Column("e", "next");
-
- static Column newNotificationColumn(String queueId) {
- return new Column(NOTIFICATION_CF, NOTIFICATION_CQ_PREFIX + queueId);
- }
-
- private final TypedTransactionBase ttx;
- private final String qid;
- private final Bytes bucketRow;
-
- static Bytes generateBucketRow(String qid, int bucket, int numBuckets) {
- return Bytes.of(qid + ":" + BucketUtil.genBucketId(bucket, numBuckets));
- }
-
- ExportBucket(TransactionBase tx, String qid, int bucket, int numBuckets) {
- // TODO encode in a more robust way... but for now fail early
- Preconditions.checkArgument(!qid.contains(":"), "Export QID can not contain :");
- this.ttx = new TypeLayer(new StringEncoder()).wrap(tx);
- this.qid = qid;
- this.bucketRow = generateBucketRow(qid, bucket, numBuckets);
- }
-
- ExportBucket(TransactionBase tx, Bytes bucketRow) {
- this.ttx = new TypeLayer(new StringEncoder()).wrap(tx);
-
- int colonLoc = -1;
-
- for (int i = 0; i < bucketRow.length(); i++) {
- if (bucketRow.byteAt(i) == ':') {
- colonLoc = i;
- break;
- }
- }
-
- Preconditions.checkArgument(colonLoc != -1 && colonLoc != bucketRow.length(),
- "Invalid bucket row " + bucketRow);
- Preconditions.checkArgument(bucketRow.byteAt(bucketRow.length() - 1) == ':',
- "Invalid bucket row " + bucketRow);
-
- this.bucketRow = bucketRow.subSequence(0, bucketRow.length() - 1);
- this.qid = bucketRow.subSequence(0, colonLoc).toString();
- }
-
- private static byte[] encSeq(long l) {
- byte[] ret = new byte[8];
- ret[0] = (byte) (l >>> 56);
- ret[1] = (byte) (l >>> 48);
- ret[2] = (byte) (l >>> 40);
- ret[3] = (byte) (l >>> 32);
- ret[4] = (byte) (l >>> 24);
- ret[5] = (byte) (l >>> 16);
- ret[6] = (byte) (l >>> 8);
- ret[7] = (byte) (l >>> 0);
- return ret;
- }
-
- private static long decodeSeq(Bytes seq) {
- return (((long) seq.byteAt(0) << 56) + ((long) (seq.byteAt(1) & 255) << 48)
- + ((long) (seq.byteAt(2) & 255) << 40) + ((long) (seq.byteAt(3) & 255) << 32)
- + ((long) (seq.byteAt(4) & 255) << 24) + ((seq.byteAt(5) & 255) << 16)
- + ((seq.byteAt(6) & 255) << 8) + ((seq.byteAt(7) & 255) << 0));
- }
-
-
- public void add(long seq, byte[] key, byte[] value) {
- Bytes row =
- Bytes.newBuilder(bucketRow.length() + 1 + key.length + 8).append(bucketRow).append(":")
- .append(key).append(encSeq(seq)).toBytes();
- ttx.set(row, EXPORT_COL, Bytes.of(value));
- }
-
- /**
- * Computes the minimial row for a bucket
- */
- private Bytes getMinimalRow() {
- return Bytes.newBuilder(bucketRow.length() + 1).append(bucketRow).append(":").toBytes();
- }
-
- public void notifyExportObserver() {
- ttx.mutate().row(getMinimalRow()).col(newNotificationColumn(qid)).weaklyNotify();
- }
-
- public Iterator<ExportEntry> getExportIterator(Bytes continueRow) {
- ScannerConfiguration sc = new ScannerConfiguration();
-
- if (continueRow != null) {
- Span tmpSpan = Span.prefix(bucketRow);
- Span nextSpan =
- new Span(new RowColumn(continueRow, EXPORT_COL), true, tmpSpan.getEnd(),
- tmpSpan.isEndInclusive());
- sc.setSpan(nextSpan);
- } else {
- sc.setSpan(Span.prefix(bucketRow));
- }
-
- sc.fetchColumn(EXPORT_COL.getFamily(), EXPORT_COL.getQualifier());
- RowIterator iter = ttx.get(sc);
-
- if (iter.hasNext()) {
- return new ExportIterator(iter);
- } else {
- return Collections.<ExportEntry>emptySet().iterator();
- }
- }
-
- private class ExportIterator implements Iterator<ExportEntry> {
-
- private RowIterator rowIter;
- private Bytes lastRow;
-
- public ExportIterator(RowIterator rowIter) {
- this.rowIter = rowIter;
- }
-
- @Override
- public boolean hasNext() {
- return rowIter.hasNext();
- }
-
- @Override
- public ExportEntry next() {
- Entry<Bytes, ColumnIterator> rowCol = rowIter.next();
- Bytes row = rowCol.getKey();
-
- Bytes keyBytes = row.subSequence(bucketRow.length() + 1, row.length() - 8);
- Bytes seqBytes = row.subSequence(row.length() - 8, row.length());
-
- ExportEntry ee = new ExportEntry();
-
- ee.key = keyBytes.toArray();
- ee.seq = decodeSeq(seqBytes);
- // TODO maybe leave as Bytes?
- ee.value = rowCol.getValue().next().getValue().toArray();
-
- lastRow = row;
-
- return ee;
- }
-
- @Override
- public void remove() {
- ttx.mutate().row(lastRow).col(EXPORT_COL).delete();
- }
- }
-
- public Bytes getContinueRow() {
- return ttx.get(getMinimalRow(), NEXT_COL);
- }
-
- public void setContinueRow(ExportEntry ee) {
- Bytes nextRow =
- Bytes.newBuilder(bucketRow.length() + 1 + ee.key.length + 8).append(bucketRow).append(":")
- .append(ee.key).append(encSeq(ee.seq)).toBytes();
-
- ttx.set(getMinimalRow(), NEXT_COL, nextRow);
- }
-
- public void clearContinueRow() {
- ttx.delete(getMinimalRow(), NEXT_COL);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportEntry.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportEntry.java
deleted file mode 100644
index 1b156b9..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportEntry.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-class ExportEntry {
- byte[] key;
- long seq;
- byte[] value;
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportObserver.java
deleted file mode 100644
index 972af6e..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportObserver.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import com.google.common.collect.Iterators;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.observer.AbstractObserver;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-
-public class ExportObserver<K, V> extends AbstractObserver {
-
- private static class MemLimitIterator implements Iterator<ExportEntry> {
-
- private long memConsumed = 0;
- private long memLimit;
- private int extraPerKey;
- private Iterator<ExportEntry> source;
-
- public MemLimitIterator(Iterator<ExportEntry> input, long limit, int extraPerKey) {
- this.source = input;
- this.memLimit = limit;
- this.extraPerKey = extraPerKey;
- }
-
- @Override
- public boolean hasNext() {
- return memConsumed < memLimit && source.hasNext();
- }
-
- @Override
- public ExportEntry next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- ExportEntry ee = source.next();
- memConsumed += ee.key.length + extraPerKey + ee.value.length;
- return ee;
- }
-
- @Override
- public void remove() {
- source.remove();
- }
- }
-
- private String queueId;
- private Class<K> keyType;
- private Class<V> valType;
- SimpleSerializer serializer;
- private Exporter<K, V> exporter;
-
- private long memLimit;
-
- protected String getQueueId() {
- return queueId;
- }
-
- SimpleSerializer getSerializer() {
- return serializer;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void init(Context context) throws Exception {
- queueId = context.getParameters().get("queueId");
- ExportQueue.Options opts = new ExportQueue.Options(queueId, context.getAppConfiguration());
-
- // TODO defer loading classes... so that not done during fluo init
- // TODO move class loading to centralized place... also attempt to check type params
- keyType = (Class<K>) getClass().getClassLoader().loadClass(opts.keyType);
- valType = (Class<V>) getClass().getClassLoader().loadClass(opts.valueType);
- exporter =
- getClass().getClassLoader().loadClass(opts.exporterType).asSubclass(Exporter.class)
- .newInstance();
-
- serializer = SimpleSerializer.getInstance(context.getAppConfiguration());
-
- memLimit = opts.getBufferSize();
-
- exporter.init(queueId, context);
- }
-
- @Override
- public ObservedColumn getObservedColumn() {
- return new ObservedColumn(ExportBucket.newNotificationColumn(queueId), NotificationType.WEAK);
- }
-
- @Override
- public void process(TransactionBase tx, Bytes row, Column column) throws Exception {
- ExportBucket bucket = new ExportBucket(tx, row);
-
- Bytes continueRow = bucket.getContinueRow();
-
- Iterator<ExportEntry> input = bucket.getExportIterator(continueRow);
- MemLimitIterator memLimitIter = new MemLimitIterator(input, memLimit, 8 + queueId.length());
-
- Iterator<SequencedExport<K, V>> exportIterator =
- Iterators.transform(
- memLimitIter,
- ee -> new SequencedExport<>(serializer.deserialize(ee.key, keyType), serializer
- .deserialize(ee.value, valType), ee.seq));
-
- exportIterator = Iterators.consumingIterator(exportIterator);
-
- exporter.processExports(exportIterator);
-
- if (input.hasNext()) {
- // not everything was processed so notify self
- bucket.notifyExportObserver();
-
- if (!memLimitIter.hasNext()) {
- // stopped because of mem limit... set continue key
- bucket.setContinueRow(input.next());
- continueRow = null;
- }
- }
-
- if (continueRow != null) {
- bucket.clearContinueRow();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportQueue.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportQueue.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportQueue.java
deleted file mode 100644
index 13518e7..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/ExportQueue.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import com.google.common.hash.Hashing;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.ObserverConfiguration;
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.recipes.common.Pirtos;
-import org.apache.fluo.recipes.common.RowRange;
-import org.apache.fluo.recipes.common.TransientRegistry;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-
-public class ExportQueue<K, V> {
-
- private static final String RANGE_BEGIN = "#";
- private static final String RANGE_END = ":~";
-
- private int numBuckets;
- private SimpleSerializer serializer;
- private String queueId;
-
- // usage hint : could be created once in an observers init method
- // usage hint : maybe have a queue for each type of data being exported???
- // maybe less queues are
- // more efficient though because more batching at export time??
- ExportQueue(Options opts, SimpleSerializer serializer) throws Exception {
- // TODO sanity check key type based on type params
- // TODO defer creating classes until needed.. so that its not done during Fluo init
- this.queueId = opts.queueId;
- this.numBuckets = opts.numBuckets;
- this.serializer = serializer;
- }
-
- public void add(TransactionBase tx, K key, V value) {
- addAll(tx, Collections.singleton(new Export<>(key, value)).iterator());
- }
-
- public void addAll(TransactionBase tx, Iterator<Export<K, V>> exports) {
-
- Set<Integer> bucketsNotified = new HashSet<>();
- while (exports.hasNext()) {
- Export<K, V> export = exports.next();
-
- byte[] k = serializer.serialize(export.getKey());
- byte[] v = serializer.serialize(export.getValue());
-
- int hash = Hashing.murmur3_32().hashBytes(k).asInt();
- int bucketId = Math.abs(hash % numBuckets);
-
- ExportBucket bucket = new ExportBucket(tx, queueId, bucketId, numBuckets);
- bucket.add(tx.getStartTimestamp(), k, v);
-
- if (!bucketsNotified.contains(bucketId)) {
- bucket.notifyExportObserver();
- bucketsNotified.add(bucketId);
- }
- }
- }
-
- public static <K2, V2> ExportQueue<K2, V2> getInstance(String exportQueueId,
- SimpleConfiguration appConfig) {
- Options opts = new Options(exportQueueId, appConfig);
- try {
- return new ExportQueue<>(opts, SimpleSerializer.getInstance(appConfig));
- } catch (Exception e) {
- // TODO
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Call this method before initializing Fluo.
- *
- * @param fluoConfig The configuration that will be used to initialize fluo.
- */
- public static void configure(FluoConfiguration fluoConfig, Options opts) {
- SimpleConfiguration appConfig = fluoConfig.getAppConfiguration();
- opts.save(appConfig);
-
- fluoConfig.addObserver(new ObserverConfiguration(ExportObserver.class.getName())
- .setParameters(Collections.singletonMap("queueId", opts.queueId)));
-
- Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
- Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
-
- new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("exportQueue."
- + opts.queueId, new RowRange(exportRangeStart, exportRangeStop));
- }
-
- /**
- * Return suggested Fluo table optimizations for all previously configured export queues.
- *
- * @param appConfig Must pass in the application configuration obtained from
- * {@code FluoClient.getAppConfiguration()} or
- * {@code FluoConfiguration.getAppConfiguration()}
- */
-
- public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
- HashSet<String> queueIds = new HashSet<>();
- appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
- k -> queueIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
-
- Pirtos pirtos = new Pirtos();
- queueIds.forEach(qid -> pirtos.merge(getTableOptimizations(qid, appConfig)));
-
- return pirtos;
- }
-
- /**
- * Return suggested Fluo table optimizations for the specified export queue.
- *
- * @param appConfig Must pass in the application configuration obtained from
- * {@code FluoClient.getAppConfiguration()} or
- * {@code FluoConfiguration.getAppConfiguration()}
- */
- public static Pirtos getTableOptimizations(String queueId, SimpleConfiguration appConfig) {
- Options opts = new Options(queueId, appConfig);
-
- List<Bytes> splits = new ArrayList<>();
-
- Bytes exportRangeStart = Bytes.of(opts.queueId + RANGE_BEGIN);
- Bytes exportRangeStop = Bytes.of(opts.queueId + RANGE_END);
-
- splits.add(exportRangeStart);
- splits.add(exportRangeStop);
-
- List<Bytes> exportSplits = new ArrayList<>();
- for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
- exportSplits.add(ExportBucket.generateBucketRow(opts.queueId, i, opts.numBuckets));
- }
- Collections.sort(exportSplits);
- splits.addAll(exportSplits);
-
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
-
- // the tablet with end row <queueId># does not contain any data for the export queue and
- // should not be grouped with the export queue
- pirtos.setTabletGroupingRegex(Pattern.quote(queueId + ":"));
-
- return pirtos;
- }
-
- public static class Options {
-
- private static final String PREFIX = "recipes.exportQueue.";
- static final long DEFAULT_BUFFER_SIZE = 1 << 20;
- static final int DEFAULT_BUCKETS_PER_TABLET = 10;
-
- int numBuckets;
- Integer bucketsPerTablet = null;
- Long bufferSize;
-
- String keyType;
- String valueType;
- String exporterType;
- String queueId;
-
- Options(String queueId, SimpleConfiguration appConfig) {
- this.queueId = queueId;
-
- this.numBuckets = appConfig.getInt(PREFIX + queueId + ".buckets");
- this.exporterType = appConfig.getString(PREFIX + queueId + ".exporter");
- this.keyType = appConfig.getString(PREFIX + queueId + ".key");
- this.valueType = appConfig.getString(PREFIX + queueId + ".val");
- this.bufferSize = appConfig.getLong(PREFIX + queueId + ".bufferSize", DEFAULT_BUFFER_SIZE);
- this.bucketsPerTablet =
- appConfig.getInt(PREFIX + queueId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
- }
-
- public Options(String queueId, String exporterType, String keyType, String valueType,
- int buckets) {
- Preconditions.checkArgument(buckets > 0);
-
- this.queueId = queueId;
- this.numBuckets = buckets;
- this.exporterType = exporterType;
- this.keyType = keyType;
- this.valueType = valueType;
- }
-
-
- public <K, V> Options(String queueId, Class<? extends Exporter<K, V>> exporter,
- Class<K> keyType, Class<V> valueType, int buckets) {
- this(queueId, exporter.getName(), keyType.getName(), valueType.getName(), buckets);
- }
-
- /**
- * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
- * be used to actually deserialize and process the updates. This limit does not account for
- * object overhead in java, which can be significant.
- *
- * <p>
- * The way memory read is calculated is by summing the length of serialized key and value byte
- * arrays. Once this sum exceeds the configured memory limit, no more export key values are
- * processed in the current transaction. When not everything is processed, the observer
- * processing exports will notify itself causing another transaction to continue processing
- * later.
- */
- public Options setBufferSize(long bufferSize) {
- Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
- this.bufferSize = bufferSize;
- return this;
- }
-
- long getBufferSize() {
- if (bufferSize == null) {
- return DEFAULT_BUFFER_SIZE;
- }
-
- return bufferSize;
- }
-
- /**
- * Sets the number of buckets per tablet to generate. This affects how many split points will be
- * generated when optimizing the Accumulo table.
- *
- */
- public Options setBucketsPerTablet(int bucketsPerTablet) {
- Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
- + bucketsPerTablet);
- this.bucketsPerTablet = bucketsPerTablet;
- return this;
- }
-
- int getBucketsPerTablet() {
- if (bucketsPerTablet == null) {
- return DEFAULT_BUCKETS_PER_TABLET;
- }
-
- return bucketsPerTablet;
- }
-
- void save(SimpleConfiguration appConfig) {
- appConfig.setProperty(PREFIX + queueId + ".buckets", numBuckets + "");
- appConfig.setProperty(PREFIX + queueId + ".exporter", exporterType + "");
- appConfig.setProperty(PREFIX + queueId + ".key", keyType);
- appConfig.setProperty(PREFIX + queueId + ".val", valueType);
-
- if (bufferSize != null) {
- appConfig.setProperty(PREFIX + queueId + ".bufferSize", bufferSize);
- }
- if (bucketsPerTablet != null) {
- appConfig.setProperty(PREFIX + queueId + ".bucketsPerTablet", bucketsPerTablet);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/Exporter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/Exporter.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/Exporter.java
deleted file mode 100644
index b81e9d1..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/Exporter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-import java.util.Iterator;
-
-import org.apache.fluo.api.observer.Observer.Context;
-
-public abstract class Exporter<K, V> {
-
- public void init(String queueId, Context observerContext) throws Exception {}
-
- /**
- * Must be able to handle same key being exported multiple times and key being exported out of
- * order. The sequence number is meant to help with this.
- *
- * <p>
- * If multiple export entries with the same key are passed in, then the entries with the same key
- * will be consecutive and in ascending sequence order.
- *
- * <p>
- * If the call to process exports is unexpectedly terminated, it will be called again later with
- * at least the same data. For example suppose an exporter was passed the following entries.
- *
- * <ul>
- * <li>key=0 sequence=9 value=abc
- * <li>key=1 sequence=13 value=d
- * <li>key=1 sequence=17 value=e
- * <li>key=1 sequence=23 value=f
- * <li>key=2 sequence=19 value=x
- * </ul>
- *
- * <p>
- * Assume the exporter exports some of these and then fails before completing all of them. The
- * next time its called it will be passed what it saw before, but it could also be passed more.
- *
- * <ul>
- * <li>key=0 sequence=9 value=abc
- * <li>key=1 sequence=13 value=d
- * <li>key=1 sequence=17 value=e
- * <li>key=1 sequence=23 value=f
- * <li>key=1 sequence=29 value=g
- * <li>key=2 sequence=19 value=x
- * <li>key=2 sequence=77 value=y
- * </ul>
- *
- */
- protected abstract void processExports(Iterator<SequencedExport<K, V>> exports);
-
- // TODO add close
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/export/SequencedExport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/export/SequencedExport.java b/modules/core/src/main/java/org/apache/fluo/recipes/export/SequencedExport.java
deleted file mode 100644
index a862a8e..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/export/SequencedExport.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.export;
-
-public class SequencedExport<K, V> extends Export<K, V> {
- private final long seq;
-
- SequencedExport(K k, V v, long seq) {
- super(k, v);
- this.seq = seq;
- }
-
- public long getSequence() {
- return seq;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/impl/BucketUtil.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/impl/BucketUtil.java b/modules/core/src/main/java/org/apache/fluo/recipes/impl/BucketUtil.java
deleted file mode 100644
index ded289c..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/impl/BucketUtil.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.impl;
-
-public class BucketUtil {
- public static String genBucketId(int bucket, int maxBucket) {
- int bucketLen = Integer.toHexString(maxBucket).length();
- // TODO printf is slow
- return String.format("%0" + bucketLen + "x", bucket);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMap.java b/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMap.java
deleted file mode 100644
index bc7bffd..0000000
--- a/modules/core/src/main/java/org/apache/fluo/recipes/map/CollisionFreeMap.java
+++ /dev/null
@@ -1,657 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.fluo.recipes.map;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-import org.apache.fluo.api.client.SnapshotBase;
-import org.apache.fluo.api.client.TransactionBase;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.api.config.ObserverConfiguration;
-import org.apache.fluo.api.config.ScannerConfiguration;
-import org.apache.fluo.api.config.SimpleConfiguration;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.BytesBuilder;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumn;
-import org.apache.fluo.api.data.RowColumnValue;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.iterator.ColumnIterator;
-import org.apache.fluo.api.iterator.RowIterator;
-import org.apache.fluo.recipes.common.Pirtos;
-import org.apache.fluo.recipes.common.RowRange;
-import org.apache.fluo.recipes.common.TransientRegistry;
-import org.apache.fluo.recipes.impl.BucketUtil;
-import org.apache.fluo.recipes.serialization.SimpleSerializer;
-
-/**
- * See the project level documentation for information about this recipe.
- */
-public class CollisionFreeMap<K, V> {
-
- private static final String UPDATE_RANGE_END = ":u:~";
-
- private static final String DATA_RANGE_END = ":d:~";
-
- private String mapId;
-
- private Class<K> keyType;
- private Class<V> valType;
- private SimpleSerializer serializer;
- private Combiner<K, V> combiner;
- UpdateObserver<K, V> updateObserver;
- private long bufferSize;
-
- static final Column UPDATE_COL = new Column("u", "v");
- static final Column NEXT_COL = new Column("u", "next");
-
- private int numBuckets = -1;
-
- @SuppressWarnings("unchecked")
- CollisionFreeMap(Options opts, SimpleSerializer serializer) throws Exception {
-
- this.mapId = opts.mapId;
- // TODO defer loading classes
- // TODO centralize class loading
- // TODO try to check type params
- this.numBuckets = opts.numBuckets;
- this.keyType = (Class<K>) getClass().getClassLoader().loadClass(opts.keyType);
- this.valType = (Class<V>) getClass().getClassLoader().loadClass(opts.valueType);
- this.combiner =
- (Combiner<K, V>) getClass().getClassLoader().loadClass(opts.combinerType).newInstance();
- this.serializer = serializer;
- if (opts.updateObserverType != null) {
- this.updateObserver =
- getClass().getClassLoader().loadClass(opts.updateObserverType)
- .asSubclass(UpdateObserver.class).newInstance();
- } else {
- this.updateObserver = new NullUpdateObserver<>();
- }
- this.bufferSize = opts.getBufferSize();
- }
-
- private V deserVal(Bytes val) {
- return serializer.deserialize(val.toArray(), valType);
- }
-
- private Bytes getKeyFromUpdateRow(Bytes prefix, Bytes row) {
- return row.subSequence(prefix.length(), row.length() - 8);
- }
-
- void process(TransactionBase tx, Bytes ntfyRow, Column col) throws Exception {
-
- Bytes nextKey = tx.get(ntfyRow, NEXT_COL);
-
- ScannerConfiguration sc = new ScannerConfiguration();
-
- if (nextKey != null) {
- Bytes startRow =
- Bytes.newBuilder(ntfyRow.length() + nextKey.length()).append(ntfyRow).append(nextKey)
- .toBytes();
- Span tmpSpan = Span.prefix(ntfyRow);
- Span nextSpan =
- new Span(new RowColumn(startRow, UPDATE_COL), false, tmpSpan.getEnd(),
- tmpSpan.isEndInclusive());
- sc.setSpan(nextSpan);
- } else {
- sc.setSpan(Span.prefix(ntfyRow));
- }
-
- sc.setSpan(Span.prefix(ntfyRow));
- sc.fetchColumn(UPDATE_COL.getFamily(), UPDATE_COL.getQualifier());
- RowIterator iter = tx.get(sc);
-
- Map<Bytes, List<Bytes>> updates = new HashMap<>();
-
- long approxMemUsed = 0;
-
- Bytes partiallyReadKey = null;
-
- if (iter.hasNext()) {
- Bytes lastKey = null;
- while (iter.hasNext() && approxMemUsed < bufferSize) {
- Entry<Bytes, ColumnIterator> rowCol = iter.next();
- Bytes curRow = rowCol.getKey();
-
- tx.delete(curRow, UPDATE_COL);
-
- Bytes serializedKey = getKeyFromUpdateRow(ntfyRow, curRow);
- lastKey = serializedKey;
-
- List<Bytes> updateList = updates.get(serializedKey);
- if (updateList == null) {
- updateList = new ArrayList<>();
- updates.put(serializedKey, updateList);
- }
-
- Bytes val = rowCol.getValue().next().getValue();
- updateList.add(val);
-
- approxMemUsed += curRow.length();
- approxMemUsed += val.length();
- }
-
- if (iter.hasNext()) {
- Entry<Bytes, ColumnIterator> rowCol = iter.next();
- Bytes curRow = rowCol.getKey();
-
- // check if more updates for last key
- if (getKeyFromUpdateRow(ntfyRow, curRow).equals(lastKey)) {
- // there are still more updates for this key
- partiallyReadKey = lastKey;
-
- // start next time at the current key
- tx.set(ntfyRow, NEXT_COL, partiallyReadKey);
- } else {
- // start next time at the next possible key
- Bytes nextPossible =
- Bytes.newBuilder(lastKey.length() + 1).append(lastKey).append(new byte[] {0})
- .toBytes();
- tx.set(ntfyRow, NEXT_COL, nextPossible);
- }
-
- // may not read all data because of mem limit, so notify self
- tx.setWeakNotification(ntfyRow, col);
- } else if (nextKey != null) {
- // clear nextKey
- tx.delete(ntfyRow, NEXT_COL);
- }
- } else if (nextKey != null) {
- tx.delete(ntfyRow, NEXT_COL);
- }
-
- byte[] dataPrefix = ntfyRow.toArray();
- // TODO this is awful... no sanity check... hard to read
- dataPrefix[Bytes.of(mapId).length() + 1] = 'd';
-
- BytesBuilder rowBuilder = Bytes.newBuilder();
- rowBuilder.append(dataPrefix);
- int rowPrefixLen = rowBuilder.getLength();
-
- Set<Bytes> keysToFetch = updates.keySet();
- if (partiallyReadKey != null) {
- final Bytes prk = partiallyReadKey;
- keysToFetch = Sets.filter(keysToFetch, b -> !b.equals(prk));
- }
- Map<Bytes, Map<Column, Bytes>> currentVals = getCurrentValues(tx, rowBuilder, keysToFetch);
-
- ArrayList<Update<K, V>> updatesToReport = new ArrayList<>(updates.size());
-
- for (Entry<Bytes, List<Bytes>> entry : updates.entrySet()) {
- rowBuilder.setLength(rowPrefixLen);
- Bytes currentValueRow = rowBuilder.append(entry.getKey()).toBytes();
- Bytes currVal =
- currentVals.getOrDefault(currentValueRow, Collections.emptyMap()).get(DATA_COLUMN);
-
- Iterator<V> ui = Iterators.transform(entry.getValue().iterator(), this::deserVal);
-
- K kd = serializer.deserialize(entry.getKey().toArray(), keyType);
-
- if (partiallyReadKey != null && partiallyReadKey.equals(entry.getKey())) {
- // not all updates were read for this key, so requeue the combined updates as an update
- Optional<V> nv = combiner.combine(kd, ui);
- if (nv.isPresent()) {
- update(tx, Collections.singletonMap(kd, nv.get()));
- }
- } else {
- Optional<V> nv = combiner.combine(kd, concat(ui, currVal));
- Bytes newVal = nv.isPresent() ? Bytes.of(serializer.serialize(nv.get())) : null;
- if (newVal != null ^ currVal != null || (currVal != null && !currVal.equals(newVal))) {
- if (newVal == null) {
- tx.delete(currentValueRow, DATA_COLUMN);
- } else {
- tx.set(currentValueRow, DATA_COLUMN, newVal);
- }
-
- Optional<V> cvd = Optional.ofNullable(currVal).map(this::deserVal);
- updatesToReport.add(new Update<>(kd, cvd, nv));
- }
- }
- }
-
- // TODO could clear these as converted to objects to avoid double memory usage
- updates.clear();
- currentVals.clear();
-
- if (updatesToReport.size() > 0) {
- updateObserver.updatingValues(tx, updatesToReport.iterator());
- }
- }
-
- private static final Column DATA_COLUMN = new Column("data", "current");
-
- private Map<Bytes, Map<Column, Bytes>> getCurrentValues(TransactionBase tx, BytesBuilder prefix,
- Set<Bytes> keySet) {
-
- Set<Bytes> rows = new HashSet<>();
-
- int prefixLen = prefix.getLength();
- for (Bytes key : keySet) {
- prefix.setLength(prefixLen);
- rows.add(prefix.append(key).toBytes());
- }
-
- try {
- return tx.get(rows, Collections.singleton(DATA_COLUMN));
- } catch (IllegalArgumentException e) {
- System.out.println(rows.size());
- throw e;
- }
- }
-
- private Iterator<V> concat(Iterator<V> updates, Bytes currentVal) {
- if (currentVal == null) {
- return updates;
- }
-
- return Iterators.concat(updates, Iterators.singletonIterator(deserVal(currentVal)));
- }
-
- /**
- * This method will retrieve the current value for key and any outstanding updates and combine
- * them using the configured {@link Combiner}. The result from the combiner is returned.
- */
- public V get(SnapshotBase tx, K key) {
-
- byte[] k = serializer.serialize(key);
-
- int hash = Hashing.murmur3_32().hashBytes(k).asInt();
- String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
-
-
- BytesBuilder rowBuilder = Bytes.newBuilder();
- rowBuilder.append(mapId).append(":u:").append(bucketId).append(":").append(k);
-
- ScannerConfiguration sc = new ScannerConfiguration();
- sc.setSpan(Span.prefix(rowBuilder.toBytes()));
-
- RowIterator iter = tx.get(sc);
-
- Iterator<V> ui;
-
- if (iter.hasNext()) {
- ui = Iterators.transform(iter, e -> deserVal(e.getValue().next().getValue()));
- } else {
- ui = Collections.<V>emptyList().iterator();
- }
-
- rowBuilder.setLength(mapId.length());
- rowBuilder.append(":d:").append(bucketId).append(":").append(k);
-
- Bytes dataRow = rowBuilder.toBytes();
-
- Bytes cv = tx.get(dataRow, DATA_COLUMN);
-
- if (!ui.hasNext()) {
- if (cv == null) {
- return null;
- } else {
- return deserVal(cv);
- }
- }
-
- return combiner.combine(key, concat(ui, cv)).orElse(null);
- }
-
- String getId() {
- return mapId;
- }
-
- /**
- * Queues updates for a collision free map. These updates will be made by an Observer executing
- * another transaction. This method will not collide with other transaction queuing updates for
- * the same keys.
- *
- * @param tx This transaction will be used to make the updates.
- * @param updates The keys in the map should correspond to keys in the collision free map being
- * updated. The values in the map will be queued for updating.
- */
- public void update(TransactionBase tx, Map<K, V> updates) {
- Preconditions.checkState(numBuckets > 0, "Not initialized");
-
- Set<String> buckets = new HashSet<>();
-
- BytesBuilder rowBuilder = Bytes.newBuilder();
- rowBuilder.append(mapId).append(":u:");
- int prefixLength = rowBuilder.getLength();
-
- byte[] startTs = encSeq(tx.getStartTimestamp());
-
- for (Entry<K, V> entry : updates.entrySet()) {
- byte[] k = serializer.serialize(entry.getKey());
- int hash = Hashing.murmur3_32().hashBytes(k).asInt();
- String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
-
- // reset to the common row prefix
- rowBuilder.setLength(prefixLength);
-
- Bytes row = rowBuilder.append(bucketId).append(":").append(k).append(startTs).toBytes();
- Bytes val = Bytes.of(serializer.serialize(entry.getValue()));
-
- // TODO set if not exists would be comforting here.... but
- // collisions on bucketId+key+uuid should never occur
- tx.set(row, UPDATE_COL, val);
-
- buckets.add(bucketId);
- }
-
- for (String bucketId : buckets) {
- rowBuilder.setLength(prefixLength);
- rowBuilder.append(bucketId).append(":");
-
- Bytes row = rowBuilder.toBytes();
-
- tx.setWeakNotification(row, new Column("fluoRecipes", "cfm:" + mapId));
- }
- }
-
-
- public static <K2, V2> CollisionFreeMap<K2, V2> getInstance(String mapId,
- SimpleConfiguration appConf) {
- Options opts = new Options(mapId, appConf);
- try {
- return new CollisionFreeMap<>(opts, SimpleSerializer.getInstance(appConf));
- } catch (Exception e) {
- // TODO
- throw new RuntimeException(e);
- }
- }
-
- /**
- * A @link {@link CollisionFreeMap} stores data in its own data format in the Fluo table. When
- * initializing a Fluo table with something like Map Reduce or Spark, data will need to be written
- * in this format. Thats the purpose of this method, it provide a simple class that can do this
- * conversion.
- *
- */
- public static <K2, V2> Initializer<K2, V2> getInitializer(String mapId, int numBuckets,
- SimpleSerializer serializer) {
- return new Initializer<>(mapId, numBuckets, serializer);
- }
-
-
- /**
- * @see CollisionFreeMap#getInitializer(String, int, SimpleSerializer)
- */
- public static class Initializer<K2, V2> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private String mapId;
-
- private SimpleSerializer serializer;
-
- private int numBuckets = -1;
-
- private Initializer(String mapId, int numBuckets, SimpleSerializer serializer) {
- this.mapId = mapId;
- this.numBuckets = numBuckets;
- this.serializer = serializer;
- }
-
- public RowColumnValue convert(K2 key, V2 val) {
- byte[] k = serializer.serialize(key);
- int hash = Hashing.murmur3_32().hashBytes(k).asInt();
- String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
-
- BytesBuilder bb = Bytes.newBuilder();
- Bytes row = bb.append(mapId).append(":d:").append(bucketId).append(":").append(k).toBytes();
- byte[] v = serializer.serialize(val);
-
- return new RowColumnValue(row, DATA_COLUMN, Bytes.of(v));
- }
- }
-
- public static class Options {
-
- static final long DEFAULT_BUFFER_SIZE = 1 << 22;
- static final int DEFAULT_BUCKETS_PER_TABLET = 10;
-
- int numBuckets;
- Integer bucketsPerTablet = null;
-
- Long bufferSize;
-
- String keyType;
- String valueType;
- String combinerType;
- String updateObserverType;
- String mapId;
-
- private static final String PREFIX = "recipes.cfm.";
-
- Options(String mapId, SimpleConfiguration appConfig) {
- this.mapId = mapId;
-
- this.numBuckets = appConfig.getInt(PREFIX + mapId + ".buckets");
- this.combinerType = appConfig.getString(PREFIX + mapId + ".combiner");
- this.keyType = appConfig.getString(PREFIX + mapId + ".key");
- this.valueType = appConfig.getString(PREFIX + mapId + ".val");
- this.updateObserverType = appConfig.getString(PREFIX + mapId + ".updateObserver", null);
- this.bufferSize = appConfig.getLong(PREFIX + mapId + ".bufferSize", DEFAULT_BUFFER_SIZE);
- this.bucketsPerTablet =
- appConfig.getInt(PREFIX + mapId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
- }
-
- public Options(String mapId, String combinerType, String keyType, String valType, int buckets) {
- Preconditions.checkArgument(buckets > 0);
- Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
-
- this.mapId = mapId;
- this.numBuckets = buckets;
- this.combinerType = combinerType;
- this.updateObserverType = null;
- this.keyType = keyType;
- this.valueType = valType;
- }
-
- public Options(String mapId, String combinerType, String updateObserverType, String keyType,
- String valueType, int buckets) {
- Preconditions.checkArgument(buckets > 0);
- Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
-
- this.mapId = mapId;
- this.numBuckets = buckets;
- this.combinerType = combinerType;
- this.updateObserverType = updateObserverType;
- this.keyType = keyType;
- this.valueType = valueType;
- }
-
- /**
- * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
- * be used to actually deserialize and process the updates. This limit does not account for
- * object overhead in java, which can be significant.
- *
- * <p>
- * The way memory read is calculated is by summing the length of serialized key and value byte
- * arrays. Once this sum exceeds the configured memory limit, no more update key values are
- * processed in the current transaction. When not everything is processed, the observer
- * processing updates will notify itself causing another transaction to continue processing
- * later
- */
- public Options setBufferSize(long bufferSize) {
- Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
- this.bufferSize = bufferSize;
- return this;
- }
-
- long getBufferSize() {
- if (bufferSize == null) {
- return DEFAULT_BUFFER_SIZE;
- }
-
- return bufferSize;
- }
-
- /**
- * Sets the number of buckets per tablet to generate. This affects how many split points will be
- * generated when optimizing the Accumulo table.
- *
- */
- public Options setBucketsPerTablet(int bucketsPerTablet) {
- Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
- + bucketsPerTablet);
- this.bucketsPerTablet = bucketsPerTablet;
- return this;
- }
-
- int getBucketsPerTablet() {
- if (bucketsPerTablet == null) {
- return DEFAULT_BUCKETS_PER_TABLET;
- }
-
- return bucketsPerTablet;
- }
-
- public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner, Class<K> keyType,
- Class<V> valueType, int buckets) {
- this(mapId, combiner.getName(), keyType.getName(), valueType.getName(), buckets);
- }
-
- public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner,
- Class<? extends UpdateObserver<K, V>> updateObserver, Class<K> keyType, Class<V> valueType,
- int buckets) {
- this(mapId, combiner.getName(), updateObserver.getName(), keyType.getName(), valueType
- .getName(), buckets);
- }
-
- void save(SimpleConfiguration appConfig) {
- appConfig.setProperty(PREFIX + mapId + ".buckets", numBuckets + "");
- appConfig.setProperty(PREFIX + mapId + ".combiner", combinerType + "");
- appConfig.setProperty(PREFIX + mapId + ".key", keyType);
- appConfig.setProperty(PREFIX + mapId + ".val", valueType);
- if (updateObserverType != null) {
- appConfig.setProperty(PREFIX + mapId + ".updateObserver", updateObserverType + "");
- }
- if (bufferSize != null) {
- appConfig.setProperty(PREFIX + mapId + ".bufferSize", bufferSize);
- }
- if (bucketsPerTablet != null) {
- appConfig.setProperty(PREFIX + mapId + ".bucketsPerTablet", bucketsPerTablet);
- }
- }
- }
-
- /**
- * This method configures a collision free map for use. It must be called before initializing
- * Fluo.
- */
- public static void configure(FluoConfiguration fluoConfig, Options opts) {
- opts.save(fluoConfig.getAppConfiguration());
- fluoConfig.addObserver(new ObserverConfiguration(CollisionFreeMapObserver.class.getName())
- .setParameters(ImmutableMap.of("mapId", opts.mapId)));
-
- Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
- Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
-
- new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("cfm." + opts.mapId,
- new RowRange(dataRangeEnd, updateRangeEnd));
- }
-
- /**
- * Return suggested Fluo table optimizations for all previously configured collision free maps.
- *
- * @param appConfig Must pass in the application configuration obtained from
- * {@code FluoClient.getAppConfiguration()} or
- * {@code FluoConfiguration.getAppConfiguration()}
- */
- public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
- HashSet<String> mapIds = new HashSet<>();
- appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
- k -> mapIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
-
- Pirtos pirtos = new Pirtos();
- mapIds.forEach(mid -> pirtos.merge(getTableOptimizations(mid, appConfig)));
-
- return pirtos;
- }
-
- /**
- * Return suggested Fluo table optimizations for the specified collisiong free map.
- *
- * @param appConfig Must pass in the application configuration obtained from
- * {@code FluoClient.getAppConfiguration()} or
- * {@code FluoConfiguration.getAppConfiguration()}
- */
- public static Pirtos getTableOptimizations(String mapId, SimpleConfiguration appConfig) {
- Options opts = new Options(mapId, appConfig);
-
- BytesBuilder rowBuilder = Bytes.newBuilder();
- rowBuilder.append(mapId);
-
- List<Bytes> dataSplits = new ArrayList<>();
- for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
- String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
- rowBuilder.setLength(mapId.length());
- dataSplits.add(rowBuilder.append(":d:").append(bucketId).toBytes());
- }
- Collections.sort(dataSplits);
-
- List<Bytes> updateSplits = new ArrayList<>();
- for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
- String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
- rowBuilder.setLength(mapId.length());
- updateSplits.add(rowBuilder.append(":u:").append(bucketId).toBytes());
- }
- Collections.sort(updateSplits);
-
- Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
- Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
-
- List<Bytes> splits = new ArrayList<>();
- splits.add(dataRangeEnd);
- splits.add(updateRangeEnd);
- splits.addAll(dataSplits);
- splits.addAll(updateSplits);
-
- Pirtos pirtos = new Pirtos();
- pirtos.setSplits(splits);
-
- pirtos.setTabletGroupingRegex(Pattern.quote(mapId + ":") + "[du]:");
-
- return pirtos;
- }
-
- private static byte[] encSeq(long l) {
- byte[] ret = new byte[8];
- ret[0] = (byte) (l >>> 56);
- ret[1] = (byte) (l >>> 48);
- ret[2] = (byte) (l >>> 40);
- ret[3] = (byte) (l >>> 32);
- ret[4] = (byte) (l >>> 24);
- ret[5] = (byte) (l >>> 16);
- ret[6] = (byte) (l >>> 8);
- ret[7] = (byte) (l >>> 0);
- return ret;
- }
-}
[06/10] incubator-fluo-recipes git commit: Updated package names in
core module
Posted by kt...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
new file mode 100644
index 0000000..61fec47
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMap.java
@@ -0,0 +1,657 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import com.google.common.hash.Hashing;
+import org.apache.fluo.api.client.SnapshotBase;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.ObserverConfiguration;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.config.SimpleConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.BytesBuilder;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumn;
+import org.apache.fluo.api.data.RowColumnValue;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+import org.apache.fluo.recipes.core.common.Pirtos;
+import org.apache.fluo.recipes.core.common.RowRange;
+import org.apache.fluo.recipes.core.common.TransientRegistry;
+import org.apache.fluo.recipes.core.impl.BucketUtil;
+import org.apache.fluo.recipes.core.serialization.SimpleSerializer;
+
+/**
+ * See the project level documentation for information about this recipe.
+ */
+public class CollisionFreeMap<K, V> {
+
+ private static final String UPDATE_RANGE_END = ":u:~";
+
+ private static final String DATA_RANGE_END = ":d:~";
+
+ private String mapId;
+
+ private Class<K> keyType;
+ private Class<V> valType;
+ private SimpleSerializer serializer;
+ private Combiner<K, V> combiner;
+ UpdateObserver<K, V> updateObserver;
+ private long bufferSize;
+
+ static final Column UPDATE_COL = new Column("u", "v");
+ static final Column NEXT_COL = new Column("u", "next");
+
+ private int numBuckets = -1;
+
+ @SuppressWarnings("unchecked")
+ CollisionFreeMap(Options opts, SimpleSerializer serializer) throws Exception {
+
+ this.mapId = opts.mapId;
+ // TODO defer loading classes
+ // TODO centralize class loading
+ // TODO try to check type params
+ this.numBuckets = opts.numBuckets;
+ this.keyType = (Class<K>) getClass().getClassLoader().loadClass(opts.keyType);
+ this.valType = (Class<V>) getClass().getClassLoader().loadClass(opts.valueType);
+ this.combiner =
+ (Combiner<K, V>) getClass().getClassLoader().loadClass(opts.combinerType).newInstance();
+ this.serializer = serializer;
+ if (opts.updateObserverType != null) {
+ this.updateObserver =
+ getClass().getClassLoader().loadClass(opts.updateObserverType)
+ .asSubclass(UpdateObserver.class).newInstance();
+ } else {
+ this.updateObserver = new NullUpdateObserver<>();
+ }
+ this.bufferSize = opts.getBufferSize();
+ }
+
+ private V deserVal(Bytes val) {
+ return serializer.deserialize(val.toArray(), valType);
+ }
+
+ private Bytes getKeyFromUpdateRow(Bytes prefix, Bytes row) {
+ return row.subSequence(prefix.length(), row.length() - 8);
+ }
+
+ void process(TransactionBase tx, Bytes ntfyRow, Column col) throws Exception {
+
+ Bytes nextKey = tx.get(ntfyRow, NEXT_COL);
+
+ ScannerConfiguration sc = new ScannerConfiguration();
+
+ if (nextKey != null) {
+ Bytes startRow =
+ Bytes.newBuilder(ntfyRow.length() + nextKey.length()).append(ntfyRow).append(nextKey)
+ .toBytes();
+ Span tmpSpan = Span.prefix(ntfyRow);
+ Span nextSpan =
+ new Span(new RowColumn(startRow, UPDATE_COL), false, tmpSpan.getEnd(),
+ tmpSpan.isEndInclusive());
+ sc.setSpan(nextSpan);
+ } else {
+ sc.setSpan(Span.prefix(ntfyRow));
+ }
+
+ sc.setSpan(Span.prefix(ntfyRow));
+ sc.fetchColumn(UPDATE_COL.getFamily(), UPDATE_COL.getQualifier());
+ RowIterator iter = tx.get(sc);
+
+ Map<Bytes, List<Bytes>> updates = new HashMap<>();
+
+ long approxMemUsed = 0;
+
+ Bytes partiallyReadKey = null;
+
+ if (iter.hasNext()) {
+ Bytes lastKey = null;
+ while (iter.hasNext() && approxMemUsed < bufferSize) {
+ Entry<Bytes, ColumnIterator> rowCol = iter.next();
+ Bytes curRow = rowCol.getKey();
+
+ tx.delete(curRow, UPDATE_COL);
+
+ Bytes serializedKey = getKeyFromUpdateRow(ntfyRow, curRow);
+ lastKey = serializedKey;
+
+ List<Bytes> updateList = updates.get(serializedKey);
+ if (updateList == null) {
+ updateList = new ArrayList<>();
+ updates.put(serializedKey, updateList);
+ }
+
+ Bytes val = rowCol.getValue().next().getValue();
+ updateList.add(val);
+
+ approxMemUsed += curRow.length();
+ approxMemUsed += val.length();
+ }
+
+ if (iter.hasNext()) {
+ Entry<Bytes, ColumnIterator> rowCol = iter.next();
+ Bytes curRow = rowCol.getKey();
+
+ // check if more updates for last key
+ if (getKeyFromUpdateRow(ntfyRow, curRow).equals(lastKey)) {
+ // there are still more updates for this key
+ partiallyReadKey = lastKey;
+
+ // start next time at the current key
+ tx.set(ntfyRow, NEXT_COL, partiallyReadKey);
+ } else {
+ // start next time at the next possible key
+ Bytes nextPossible =
+ Bytes.newBuilder(lastKey.length() + 1).append(lastKey).append(new byte[] {0})
+ .toBytes();
+ tx.set(ntfyRow, NEXT_COL, nextPossible);
+ }
+
+ // may not read all data because of mem limit, so notify self
+ tx.setWeakNotification(ntfyRow, col);
+ } else if (nextKey != null) {
+ // clear nextKey
+ tx.delete(ntfyRow, NEXT_COL);
+ }
+ } else if (nextKey != null) {
+ tx.delete(ntfyRow, NEXT_COL);
+ }
+
+ byte[] dataPrefix = ntfyRow.toArray();
+ // TODO this is awful... no sanity check... hard to read
+ dataPrefix[Bytes.of(mapId).length() + 1] = 'd';
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(dataPrefix);
+ int rowPrefixLen = rowBuilder.getLength();
+
+ Set<Bytes> keysToFetch = updates.keySet();
+ if (partiallyReadKey != null) {
+ final Bytes prk = partiallyReadKey;
+ keysToFetch = Sets.filter(keysToFetch, b -> !b.equals(prk));
+ }
+ Map<Bytes, Map<Column, Bytes>> currentVals = getCurrentValues(tx, rowBuilder, keysToFetch);
+
+ ArrayList<Update<K, V>> updatesToReport = new ArrayList<>(updates.size());
+
+ for (Entry<Bytes, List<Bytes>> entry : updates.entrySet()) {
+ rowBuilder.setLength(rowPrefixLen);
+ Bytes currentValueRow = rowBuilder.append(entry.getKey()).toBytes();
+ Bytes currVal =
+ currentVals.getOrDefault(currentValueRow, Collections.emptyMap()).get(DATA_COLUMN);
+
+ Iterator<V> ui = Iterators.transform(entry.getValue().iterator(), this::deserVal);
+
+ K kd = serializer.deserialize(entry.getKey().toArray(), keyType);
+
+ if (partiallyReadKey != null && partiallyReadKey.equals(entry.getKey())) {
+ // not all updates were read for this key, so requeue the combined updates as an update
+ Optional<V> nv = combiner.combine(kd, ui);
+ if (nv.isPresent()) {
+ update(tx, Collections.singletonMap(kd, nv.get()));
+ }
+ } else {
+ Optional<V> nv = combiner.combine(kd, concat(ui, currVal));
+ Bytes newVal = nv.isPresent() ? Bytes.of(serializer.serialize(nv.get())) : null;
+ if (newVal != null ^ currVal != null || (currVal != null && !currVal.equals(newVal))) {
+ if (newVal == null) {
+ tx.delete(currentValueRow, DATA_COLUMN);
+ } else {
+ tx.set(currentValueRow, DATA_COLUMN, newVal);
+ }
+
+ Optional<V> cvd = Optional.ofNullable(currVal).map(this::deserVal);
+ updatesToReport.add(new Update<>(kd, cvd, nv));
+ }
+ }
+ }
+
+ // TODO could clear these as converted to objects to avoid double memory usage
+ updates.clear();
+ currentVals.clear();
+
+ if (updatesToReport.size() > 0) {
+ updateObserver.updatingValues(tx, updatesToReport.iterator());
+ }
+ }
+
+ private static final Column DATA_COLUMN = new Column("data", "current");
+
+ private Map<Bytes, Map<Column, Bytes>> getCurrentValues(TransactionBase tx, BytesBuilder prefix,
+ Set<Bytes> keySet) {
+
+ Set<Bytes> rows = new HashSet<>();
+
+ int prefixLen = prefix.getLength();
+ for (Bytes key : keySet) {
+ prefix.setLength(prefixLen);
+ rows.add(prefix.append(key).toBytes());
+ }
+
+ try {
+ return tx.get(rows, Collections.singleton(DATA_COLUMN));
+ } catch (IllegalArgumentException e) {
+ System.out.println(rows.size());
+ throw e;
+ }
+ }
+
+ private Iterator<V> concat(Iterator<V> updates, Bytes currentVal) {
+ if (currentVal == null) {
+ return updates;
+ }
+
+ return Iterators.concat(updates, Iterators.singletonIterator(deserVal(currentVal)));
+ }
+
+ /**
+ * This method will retrieve the current value for key and any outstanding updates and combine
+ * them using the configured {@link Combiner}. The result from the combiner is returned.
+ */
+ public V get(SnapshotBase tx, K key) {
+
+ byte[] k = serializer.serialize(key);
+
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId).append(":u:").append(bucketId).append(":").append(k);
+
+ ScannerConfiguration sc = new ScannerConfiguration();
+ sc.setSpan(Span.prefix(rowBuilder.toBytes()));
+
+ RowIterator iter = tx.get(sc);
+
+ Iterator<V> ui;
+
+ if (iter.hasNext()) {
+ ui = Iterators.transform(iter, e -> deserVal(e.getValue().next().getValue()));
+ } else {
+ ui = Collections.<V>emptyList().iterator();
+ }
+
+ rowBuilder.setLength(mapId.length());
+ rowBuilder.append(":d:").append(bucketId).append(":").append(k);
+
+ Bytes dataRow = rowBuilder.toBytes();
+
+ Bytes cv = tx.get(dataRow, DATA_COLUMN);
+
+ if (!ui.hasNext()) {
+ if (cv == null) {
+ return null;
+ } else {
+ return deserVal(cv);
+ }
+ }
+
+ return combiner.combine(key, concat(ui, cv)).orElse(null);
+ }
+
+ String getId() {
+ return mapId;
+ }
+
+ /**
+ * Queues updates for a collision free map. These updates will be made by an Observer executing
+ * another transaction. This method will not collide with other transaction queuing updates for
+ * the same keys.
+ *
+ * @param tx This transaction will be used to make the updates.
+ * @param updates The keys in the map should correspond to keys in the collision free map being
+ * updated. The values in the map will be queued for updating.
+ */
+ public void update(TransactionBase tx, Map<K, V> updates) {
+ Preconditions.checkState(numBuckets > 0, "Not initialized");
+
+ Set<String> buckets = new HashSet<>();
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId).append(":u:");
+ int prefixLength = rowBuilder.getLength();
+
+ byte[] startTs = encSeq(tx.getStartTimestamp());
+
+ for (Entry<K, V> entry : updates.entrySet()) {
+ byte[] k = serializer.serialize(entry.getKey());
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+ // reset to the common row prefix
+ rowBuilder.setLength(prefixLength);
+
+ Bytes row = rowBuilder.append(bucketId).append(":").append(k).append(startTs).toBytes();
+ Bytes val = Bytes.of(serializer.serialize(entry.getValue()));
+
+ // TODO set if not exists would be comforting here.... but
+ // collisions on bucketId+key+uuid should never occur
+ tx.set(row, UPDATE_COL, val);
+
+ buckets.add(bucketId);
+ }
+
+ for (String bucketId : buckets) {
+ rowBuilder.setLength(prefixLength);
+ rowBuilder.append(bucketId).append(":");
+
+ Bytes row = rowBuilder.toBytes();
+
+ tx.setWeakNotification(row, new Column("fluoRecipes", "cfm:" + mapId));
+ }
+ }
+
+
+ public static <K2, V2> CollisionFreeMap<K2, V2> getInstance(String mapId,
+ SimpleConfiguration appConf) {
+ Options opts = new Options(mapId, appConf);
+ try {
+ return new CollisionFreeMap<>(opts, SimpleSerializer.getInstance(appConf));
+ } catch (Exception e) {
+ // TODO
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * A @link {@link CollisionFreeMap} stores data in its own data format in the Fluo table. When
+ * initializing a Fluo table with something like Map Reduce or Spark, data will need to be written
+ * in this format. Thats the purpose of this method, it provide a simple class that can do this
+ * conversion.
+ *
+ */
+ public static <K2, V2> Initializer<K2, V2> getInitializer(String mapId, int numBuckets,
+ SimpleSerializer serializer) {
+ return new Initializer<>(mapId, numBuckets, serializer);
+ }
+
+
+ /**
+ * @see CollisionFreeMap#getInitializer(String, int, SimpleSerializer)
+ */
+ public static class Initializer<K2, V2> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String mapId;
+
+ private SimpleSerializer serializer;
+
+ private int numBuckets = -1;
+
+ private Initializer(String mapId, int numBuckets, SimpleSerializer serializer) {
+ this.mapId = mapId;
+ this.numBuckets = numBuckets;
+ this.serializer = serializer;
+ }
+
+ public RowColumnValue convert(K2 key, V2 val) {
+ byte[] k = serializer.serialize(key);
+ int hash = Hashing.murmur3_32().hashBytes(k).asInt();
+ String bucketId = BucketUtil.genBucketId(Math.abs(hash % numBuckets), numBuckets);
+
+ BytesBuilder bb = Bytes.newBuilder();
+ Bytes row = bb.append(mapId).append(":d:").append(bucketId).append(":").append(k).toBytes();
+ byte[] v = serializer.serialize(val);
+
+ return new RowColumnValue(row, DATA_COLUMN, Bytes.of(v));
+ }
+ }
+
+ public static class Options {
+
+ static final long DEFAULT_BUFFER_SIZE = 1 << 22;
+ static final int DEFAULT_BUCKETS_PER_TABLET = 10;
+
+ int numBuckets;
+ Integer bucketsPerTablet = null;
+
+ Long bufferSize;
+
+ String keyType;
+ String valueType;
+ String combinerType;
+ String updateObserverType;
+ String mapId;
+
+ private static final String PREFIX = "recipes.cfm.";
+
+ Options(String mapId, SimpleConfiguration appConfig) {
+ this.mapId = mapId;
+
+ this.numBuckets = appConfig.getInt(PREFIX + mapId + ".buckets");
+ this.combinerType = appConfig.getString(PREFIX + mapId + ".combiner");
+ this.keyType = appConfig.getString(PREFIX + mapId + ".key");
+ this.valueType = appConfig.getString(PREFIX + mapId + ".val");
+ this.updateObserverType = appConfig.getString(PREFIX + mapId + ".updateObserver", null);
+ this.bufferSize = appConfig.getLong(PREFIX + mapId + ".bufferSize", DEFAULT_BUFFER_SIZE);
+ this.bucketsPerTablet =
+ appConfig.getInt(PREFIX + mapId + ".bucketsPerTablet", DEFAULT_BUCKETS_PER_TABLET);
+ }
+
+ public Options(String mapId, String combinerType, String keyType, String valType, int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+ Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
+
+ this.mapId = mapId;
+ this.numBuckets = buckets;
+ this.combinerType = combinerType;
+ this.updateObserverType = null;
+ this.keyType = keyType;
+ this.valueType = valType;
+ }
+
+ public Options(String mapId, String combinerType, String updateObserverType, String keyType,
+ String valueType, int buckets) {
+ Preconditions.checkArgument(buckets > 0);
+ Preconditions.checkArgument(!mapId.contains(":"), "Map id cannot contain ':'");
+
+ this.mapId = mapId;
+ this.numBuckets = buckets;
+ this.combinerType = combinerType;
+ this.updateObserverType = updateObserverType;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ /**
+ * Sets a limit on the amount of serialized updates to read into memory. Additional memory will
+ * be used to actually deserialize and process the updates. This limit does not account for
+ * object overhead in java, which can be significant.
+ *
+ * <p>
+ * The way memory read is calculated is by summing the length of serialized key and value byte
+ * arrays. Once this sum exceeds the configured memory limit, no more update key values are
+ * processed in the current transaction. When not everything is processed, the observer
+ * processing updates will notify itself causing another transaction to continue processing
+ * later
+ */
+ public Options setBufferSize(long bufferSize) {
+ Preconditions.checkArgument(bufferSize > 0, "Buffer size must be positive");
+ this.bufferSize = bufferSize;
+ return this;
+ }
+
+ long getBufferSize() {
+ if (bufferSize == null) {
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ return bufferSize;
+ }
+
+ /**
+ * Sets the number of buckets per tablet to generate. This affects how many split points will be
+ * generated when optimizing the Accumulo table.
+ *
+ */
+ public Options setBucketsPerTablet(int bucketsPerTablet) {
+ Preconditions.checkArgument(bucketsPerTablet > 0, "bucketsPerTablet is <= 0 : "
+ + bucketsPerTablet);
+ this.bucketsPerTablet = bucketsPerTablet;
+ return this;
+ }
+
+ int getBucketsPerTablet() {
+ if (bucketsPerTablet == null) {
+ return DEFAULT_BUCKETS_PER_TABLET;
+ }
+
+ return bucketsPerTablet;
+ }
+
+ public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner, Class<K> keyType,
+ Class<V> valueType, int buckets) {
+ this(mapId, combiner.getName(), keyType.getName(), valueType.getName(), buckets);
+ }
+
+ public <K, V> Options(String mapId, Class<? extends Combiner<K, V>> combiner,
+ Class<? extends UpdateObserver<K, V>> updateObserver, Class<K> keyType, Class<V> valueType,
+ int buckets) {
+ this(mapId, combiner.getName(), updateObserver.getName(), keyType.getName(), valueType
+ .getName(), buckets);
+ }
+
+ void save(SimpleConfiguration appConfig) {
+ appConfig.setProperty(PREFIX + mapId + ".buckets", numBuckets + "");
+ appConfig.setProperty(PREFIX + mapId + ".combiner", combinerType + "");
+ appConfig.setProperty(PREFIX + mapId + ".key", keyType);
+ appConfig.setProperty(PREFIX + mapId + ".val", valueType);
+ if (updateObserverType != null) {
+ appConfig.setProperty(PREFIX + mapId + ".updateObserver", updateObserverType + "");
+ }
+ if (bufferSize != null) {
+ appConfig.setProperty(PREFIX + mapId + ".bufferSize", bufferSize);
+ }
+ if (bucketsPerTablet != null) {
+ appConfig.setProperty(PREFIX + mapId + ".bucketsPerTablet", bucketsPerTablet);
+ }
+ }
+ }
+
+ /**
+ * This method configures a collision free map for use. It must be called before initializing
+ * Fluo.
+ */
+ public static void configure(FluoConfiguration fluoConfig, Options opts) {
+ opts.save(fluoConfig.getAppConfiguration());
+ fluoConfig.addObserver(new ObserverConfiguration(CollisionFreeMapObserver.class.getName())
+ .setParameters(ImmutableMap.of("mapId", opts.mapId)));
+
+ Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
+ Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
+
+ new TransientRegistry(fluoConfig.getAppConfiguration()).addTransientRange("cfm." + opts.mapId,
+ new RowRange(dataRangeEnd, updateRangeEnd));
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for all previously configured collision free maps.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
+ public static Pirtos getTableOptimizations(SimpleConfiguration appConfig) {
+ HashSet<String> mapIds = new HashSet<>();
+ appConfig.getKeys(Options.PREFIX.substring(0, Options.PREFIX.length() - 1)).forEachRemaining(
+ k -> mapIds.add(k.substring(Options.PREFIX.length()).split("\\.", 2)[0]));
+
+ Pirtos pirtos = new Pirtos();
+ mapIds.forEach(mid -> pirtos.merge(getTableOptimizations(mid, appConfig)));
+
+ return pirtos;
+ }
+
+ /**
+ * Return suggested Fluo table optimizations for the specified collisiong free map.
+ *
+ * @param appConfig Must pass in the application configuration obtained from
+ * {@code FluoClient.getAppConfiguration()} or
+ * {@code FluoConfiguration.getAppConfiguration()}
+ */
+ public static Pirtos getTableOptimizations(String mapId, SimpleConfiguration appConfig) {
+ Options opts = new Options(mapId, appConfig);
+
+ BytesBuilder rowBuilder = Bytes.newBuilder();
+ rowBuilder.append(mapId);
+
+ List<Bytes> dataSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
+ rowBuilder.setLength(mapId.length());
+ dataSplits.add(rowBuilder.append(":d:").append(bucketId).toBytes());
+ }
+ Collections.sort(dataSplits);
+
+ List<Bytes> updateSplits = new ArrayList<>();
+ for (int i = opts.getBucketsPerTablet(); i < opts.numBuckets; i += opts.getBucketsPerTablet()) {
+ String bucketId = BucketUtil.genBucketId(i, opts.numBuckets);
+ rowBuilder.setLength(mapId.length());
+ updateSplits.add(rowBuilder.append(":u:").append(bucketId).toBytes());
+ }
+ Collections.sort(updateSplits);
+
+ Bytes dataRangeEnd = Bytes.of(opts.mapId + DATA_RANGE_END);
+ Bytes updateRangeEnd = Bytes.of(opts.mapId + UPDATE_RANGE_END);
+
+ List<Bytes> splits = new ArrayList<>();
+ splits.add(dataRangeEnd);
+ splits.add(updateRangeEnd);
+ splits.addAll(dataSplits);
+ splits.addAll(updateSplits);
+
+ Pirtos pirtos = new Pirtos();
+ pirtos.setSplits(splits);
+
+ pirtos.setTabletGroupingRegex(Pattern.quote(mapId + ":") + "[du]:");
+
+ return pirtos;
+ }
+
+ private static byte[] encSeq(long l) {
+ byte[] ret = new byte[8];
+ ret[0] = (byte) (l >>> 56);
+ ret[1] = (byte) (l >>> 48);
+ ret[2] = (byte) (l >>> 40);
+ ret[3] = (byte) (l >>> 32);
+ ret[4] = (byte) (l >>> 24);
+ ret[5] = (byte) (l >>> 16);
+ ret[6] = (byte) (l >>> 8);
+ ret[7] = (byte) (l >>> 0);
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
new file mode 100644
index 0000000..96890af
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/CollisionFreeMapObserver.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.observer.AbstractObserver;
+
+/**
+ * This class is configured for use by CollisionFreeMap.configure(FluoConfiguration,
+ * CollisionFreeMap.Options) . This class should never have to be used directly.
+ */
+
+public class CollisionFreeMapObserver extends AbstractObserver {
+
+ @SuppressWarnings("rawtypes")
+ private CollisionFreeMap cfm;
+ private String mapId;
+
+ public CollisionFreeMapObserver() {}
+
+ @Override
+ public void init(Context context) throws Exception {
+ this.mapId = context.getParameters().get("mapId");
+ cfm = CollisionFreeMap.getInstance(mapId, context.getAppConfiguration());
+ cfm.updateObserver.init(mapId, context);
+ }
+
+ @Override
+ public void process(TransactionBase tx, Bytes row, Column col) throws Exception {
+ cfm.process(tx, row, col);
+ }
+
+ @Override
+ public ObservedColumn getObservedColumn() {
+ // TODO constants
+ return new ObservedColumn(new Column("fluoRecipes", "cfm:" + mapId), NotificationType.WEAK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
new file mode 100644
index 0000000..c9d468b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Combiner.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Iterator;
+import java.util.Optional;
+
+public interface Combiner<K, V> {
+ /**
+ * This function is called to combine the current value of a key with updates that were queued for
+ * the key. See the collision free map project level documentation for more information.
+ *
+ * @return Then new value for the key. Returning Optional.absent() will cause the key to be
+ * deleted.
+ */
+
+ Optional<V> combine(K key, Iterator<V> updates);
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/NullUpdateObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/NullUpdateObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/NullUpdateObserver.java
new file mode 100644
index 0000000..7bbfec1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/NullUpdateObserver.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Iterator;
+
+import org.apache.fluo.api.client.TransactionBase;
+
+class NullUpdateObserver<K, V> extends UpdateObserver<K, V> {
+ @Override
+ public void updatingValues(TransactionBase tx, Iterator<Update<K, V>> updates) {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
new file mode 100644
index 0000000..10e718e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/Update.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Optional;
+
+public class Update<K, V> {
+
+ private final K key;
+ private final Optional<V> oldValue;
+ private final Optional<V> newValue;
+
+ Update(K key, Optional<V> oldValue, Optional<V> newValue) {
+ this.key = key;
+ this.oldValue = oldValue;
+ this.newValue = newValue;
+ }
+
+ public K getKey() {
+ return key;
+ }
+
+ public Optional<V> getNewValue() {
+ return newValue;
+ }
+
+ public Optional<V> getOldValue() {
+ return oldValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
new file mode 100644
index 0000000..2e48451
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/map/UpdateObserver.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.map;
+
+import java.util.Iterator;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.observer.Observer.Context;
+
+/**
+ * A {@link CollisionFreeMap} calls this to allow additional processing to be done when key values
+ * are updated. See the project level documentation for more information.
+ */
+
+public abstract class UpdateObserver<K, V> {
+ public void init(String mapId, Context observerContext) throws Exception {}
+
+ public abstract void updatingValues(TransactionBase tx, Iterator<Update<K, V>> updates);
+
+ // TODO add close
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
new file mode 100644
index 0000000..589b9cc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/serialization/SimpleSerializer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.serialization;
+
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.api.config.SimpleConfiguration;
+
+public interface SimpleSerializer {
+
+ /**
+ * Called immediately after construction and passed Fluo application configuration.
+ */
+ public void init(SimpleConfiguration appConfig);
+
+ // TODO refactor to support reuse of objects and byte arrays???
+ public <T> byte[] serialize(T obj);
+
+ public <T> T deserialize(byte[] serObj, Class<T> clazz);
+
+ public static void setSetserlializer(FluoConfiguration fluoConfig,
+ Class<? extends SimpleSerializer> serializerType) {
+ setSetserlializer(fluoConfig, serializerType.getName());
+ }
+
+ public static void setSetserlializer(FluoConfiguration fluoConfig, String serializerType) {
+ fluoConfig.getAppConfiguration().setProperty("recipes.serializer", serializerType);
+ }
+
+ public static SimpleSerializer getInstance(SimpleConfiguration appConfig) {
+ String serType =
+ appConfig.getString("recipes.serializer",
+ "org.apache.fluo.recipes.kryo.KryoSimplerSerializer");
+ try {
+ SimpleSerializer simplerSer =
+ SimpleSerializer.class.getClassLoader().loadClass(serType)
+ .asSubclass(SimpleSerializer.class).newInstance();
+ simplerSer.init(appConfig);
+ return simplerSer;
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
new file mode 100644
index 0000000..fa25b63
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/LogEntry.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.transaction;
+
+import java.util.Objects;
+
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+
+/**
+ * Logs an operation (i.e GET, SET, or DELETE) in a Transaction. Multiple LogEntry objects make up a
+ * {@link TxLog}.
+ */
+public class LogEntry {
+
+ public enum Operation {
+ GET, SET, DELETE
+ }
+
+ private Operation op;
+ private Bytes row;
+ private Column col;
+ private Bytes value;
+
+ private LogEntry() {}
+
+ private LogEntry(Operation op, Bytes row, Column col, Bytes value) {
+ Objects.requireNonNull(op);
+ Objects.requireNonNull(row);
+ Objects.requireNonNull(col);
+ Objects.requireNonNull(value);
+ this.op = op;
+ this.row = row;
+ this.col = col;
+ this.value = value;
+ }
+
+ public Operation getOp() {
+ return op;
+ }
+
+ public Bytes getRow() {
+ return row;
+ }
+
+ public Column getColumn() {
+ return col;
+ }
+
+ public Bytes getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof LogEntry) {
+ LogEntry other = (LogEntry) o;
+ return ((op == other.op) && row.equals(other.row) && col.equals(other.col) && value
+ .equals(other.value));
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = op.hashCode();
+ result = 31 * result + row.hashCode();
+ result = 31 * result + col.hashCode();
+ result = 31 * result + value.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "LogEntry{op=" + op + ", row=" + row + ", col=" + col + ", value=" + value + "}";
+ }
+
+ public static LogEntry newGet(String row, Column col, String value) {
+ return newGet(Bytes.of(row), col, Bytes.of(value));
+ }
+
+ public static LogEntry newGet(Bytes row, Column col, Bytes value) {
+ return new LogEntry(Operation.GET, row, col, value);
+ }
+
+ public static LogEntry newSet(String row, Column col, String value) {
+ return newSet(Bytes.of(row), col, Bytes.of(value));
+ }
+
+ public static LogEntry newSet(Bytes row, Column col, Bytes value) {
+ return new LogEntry(Operation.SET, row, col, value);
+ }
+
+ public static LogEntry newDelete(String row, Column col) {
+ return newDelete(Bytes.of(row), col);
+ }
+
+ public static LogEntry newDelete(Bytes row, Column col) {
+ return new LogEntry(Operation.DELETE, row, col, Bytes.EMPTY);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
new file mode 100644
index 0000000..8fb98d3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransaction.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.transaction;
+
+import java.util.function.Predicate;
+
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.exceptions.CommitException;
+
+/**
+ * An implementation of {@link Transaction} that logs all transactions operations (GET, SET, or
+ * DELETE) in a {@link TxLog} that can be used for exports
+ */
+public class RecordingTransaction extends RecordingTransactionBase implements Transaction {
+
+ private final Transaction tx;
+
+ private RecordingTransaction(Transaction tx) {
+ super(tx);
+ this.tx = tx;
+ }
+
+ private RecordingTransaction(Transaction tx, Predicate<LogEntry> filter) {
+ super(tx, filter);
+ this.tx = tx;
+ }
+
+ @Override
+ public void commit() throws CommitException {
+ tx.commit();
+ }
+
+ @Override
+ public void close() {
+ tx.close();
+ }
+
+ /**
+ * Creates a RecordingTransaction by wrapping an existing Transaction
+ */
+ public static RecordingTransaction wrap(Transaction tx) {
+ return new RecordingTransaction(tx);
+ }
+
+ /**
+ * Creates a RecordingTransaction using the provided LogEntry filter and existing Transaction
+ */
+ public static RecordingTransaction wrap(Transaction tx, Predicate<LogEntry> filter) {
+ return new RecordingTransaction(tx, filter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
new file mode 100644
index 0000000..6303122
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/RecordingTransactionBase.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.transaction;
+
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.config.ScannerConfiguration;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumn;
+import org.apache.fluo.api.exceptions.AlreadySetException;
+import org.apache.fluo.api.iterator.ColumnIterator;
+import org.apache.fluo.api.iterator.RowIterator;
+
+/**
+ * An implementation of {@link TransactionBase} that logs all transactions operations (GET, SET, or
+ * DELETE) in a {@link TxLog} that can be used for exports
+ */
+public class RecordingTransactionBase implements TransactionBase {
+
+ private final TransactionBase txb;
+ private final TxLog txLog = new TxLog();
+ private final Predicate<LogEntry> filter;
+
+ RecordingTransactionBase(TransactionBase txb, Predicate<LogEntry> filter) {
+ this.txb = txb;
+ this.filter = filter;
+ }
+
+ RecordingTransactionBase(TransactionBase txb) {
+ this(txb, le -> true);
+ }
+
+ @Override
+ public void setWeakNotification(Bytes row, Column col) {
+ txb.setWeakNotification(row, col);
+ }
+
+ @Override
+ public void setWeakNotification(String row, Column col) {
+ txb.setWeakNotification(row, col);
+ }
+
+ @Override
+ public void set(Bytes row, Column col, Bytes value) throws AlreadySetException {
+ txLog.filteredAdd(LogEntry.newSet(row, col, value), filter);
+ txb.set(row, col, value);
+ }
+
+ @Override
+ public void set(String row, Column col, String value) throws AlreadySetException {
+ txLog.filteredAdd(LogEntry.newSet(row, col, value), filter);
+ txb.set(row, col, value);
+ }
+
+ @Override
+ public void delete(Bytes row, Column col) {
+ txLog.filteredAdd(LogEntry.newDelete(row, col), filter);
+ txb.delete(row, col);
+ }
+
+ @Override
+ public void delete(String row, Column col) {
+ txLog.filteredAdd(LogEntry.newDelete(row, col), filter);
+ txb.delete(row, col);
+ }
+
+ /**
+ * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
+ */
+ @Override
+ public Bytes get(Bytes row, Column col) {
+ Bytes val = txb.get(row, col);
+ if (val != null) {
+ txLog.filteredAdd(LogEntry.newGet(row, col, val), filter);
+ }
+ return val;
+ }
+
+ /**
+ * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
+ */
+ @Override
+ public Map<Column, Bytes> get(Bytes row, Set<Column> columns) {
+ Map<Column, Bytes> colVal = txb.get(row, columns);
+ for (Map.Entry<Column, Bytes> entry : colVal.entrySet()) {
+ txLog.filteredAdd(LogEntry.newGet(row, entry.getKey(), entry.getValue()), filter);
+ }
+ return colVal;
+ }
+
+ /**
+ * Logs GETs for returned Row/Columns. Requests that return no data will not be logged.
+ */
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<Bytes> rows, Set<Column> columns) {
+ Map<Bytes, Map<Column, Bytes>> rowColVal = txb.get(rows, columns);
+ for (Map.Entry<Bytes, Map<Column, Bytes>> rowEntry : rowColVal.entrySet()) {
+ for (Map.Entry<Column, Bytes> colEntry : rowEntry.getValue().entrySet()) {
+ txLog.filteredAdd(
+ LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
+ }
+ }
+ return rowColVal;
+ }
+
+ @Override
+ public Map<Bytes, Map<Column, Bytes>> get(Collection<RowColumn> rowColumns) {
+ Map<Bytes, Map<Column, Bytes>> rowColVal = txb.get(rowColumns);
+ for (Map.Entry<Bytes, Map<Column, Bytes>> rowEntry : rowColVal.entrySet()) {
+ for (Map.Entry<Column, Bytes> colEntry : rowEntry.getValue().entrySet()) {
+ txLog.filteredAdd(
+ LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
+ }
+ }
+ return rowColVal;
+ }
+
+ /**
+ * Logs GETs for Row/Columns returned by iterators. Requests that return no data will not be
+ * logged.
+ */
+ @Override
+ public RowIterator get(ScannerConfiguration config) {
+ final RowIterator rowIter = txb.get(config);
+ if (rowIter != null) {
+ return new RowIterator() {
+
+ @Override
+ public boolean hasNext() {
+ return rowIter.hasNext();
+ }
+
+ @Override
+ public Map.Entry<Bytes, ColumnIterator> next() {
+ final Map.Entry<Bytes, ColumnIterator> rowEntry = rowIter.next();
+ if ((rowEntry != null) && (rowEntry.getValue() != null)) {
+ final ColumnIterator colIter = rowEntry.getValue();
+ return new AbstractMap.SimpleEntry<>(rowEntry.getKey(), new ColumnIterator() {
+
+ @Override
+ public boolean hasNext() {
+ return colIter.hasNext();
+ }
+
+ @Override
+ public Map.Entry<Column, Bytes> next() {
+ Map.Entry<Column, Bytes> colEntry = colIter.next();
+ if (colEntry != null) {
+ txLog.filteredAdd(
+ LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()),
+ filter);
+ }
+ return colEntry;
+ }
+ });
+ }
+ return rowEntry;
+ }
+ };
+ }
+ return rowIter;
+ }
+
+ @Override
+ public long getStartTimestamp() {
+ return txb.getStartTimestamp();
+ }
+
+ public TxLog getTxLog() {
+ return txLog;
+ }
+
+ /**
+ * Creates a RecordingTransactionBase by wrapping an existing TransactionBase
+ */
+ public static RecordingTransactionBase wrap(TransactionBase txb) {
+ return new RecordingTransactionBase(txb);
+ }
+
+ /**
+ * Creates a RecordingTransactionBase using the provided LogEntry filter function and existing
+ * TransactionBase
+ */
+ public static RecordingTransactionBase wrap(TransactionBase txb, Predicate<LogEntry> filter) {
+ return new RecordingTransactionBase(txb, filter);
+ }
+
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<RowColumn> rowColumns) {
+ Map<String, Map<Column, String>> rowColVal = txb.gets(rowColumns);
+ for (Map.Entry<String, Map<Column, String>> rowEntry : rowColVal.entrySet()) {
+ for (Map.Entry<Column, String> colEntry : rowEntry.getValue().entrySet()) {
+ txLog.filteredAdd(
+ LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
+ }
+ }
+ return rowColVal;
+ }
+
+ // TODO alot of these String methods may be more efficient if called the Byte version in this
+ // class... this would avoid conversion from Byte->String->Byte
+ @Override
+ public Map<String, Map<Column, String>> gets(Collection<String> rows, Set<Column> columns) {
+ Map<String, Map<Column, String>> rowColVal = txb.gets(rows, columns);
+ for (Map.Entry<String, Map<Column, String>> rowEntry : rowColVal.entrySet()) {
+ for (Map.Entry<Column, String> colEntry : rowEntry.getValue().entrySet()) {
+ txLog.filteredAdd(
+ LogEntry.newGet(rowEntry.getKey(), colEntry.getKey(), colEntry.getValue()), filter);
+ }
+ }
+ return rowColVal;
+ }
+
+ @Override
+ public String gets(String row, Column col) {
+ String val = txb.gets(row, col);
+ if (val != null) {
+ txLog.filteredAdd(LogEntry.newGet(row, col, val), filter);
+ }
+ return val;
+ }
+
+ @Override
+ public Map<Column, String> gets(String row, Set<Column> columns) {
+ Map<Column, String> colVal = txb.gets(row, columns);
+ for (Map.Entry<Column, String> entry : colVal.entrySet()) {
+ txLog.filteredAdd(LogEntry.newGet(row, entry.getKey(), entry.getValue()), filter);
+ }
+ return colVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
new file mode 100644
index 0000000..caa3c4d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/transaction/TxLog.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.transaction;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.RowColumn;
+
+/**
+ * Contains list of operations (GET, SET, DELETE) performed during a {@link RecordingTransaction}
+ */
+public class TxLog {
+
+ private List<LogEntry> logEntries = new ArrayList<>();
+
+ public TxLog() {}
+
+ /**
+ * Adds LogEntry to TxLog
+ */
+ public void add(LogEntry entry) {
+ logEntries.add(entry);
+ }
+
+ /**
+ * Adds LogEntry to TxLog if it passes filter
+ */
+ public void filteredAdd(LogEntry entry, Predicate<LogEntry> filter) {
+ if (filter.test(entry)) {
+ add(entry);
+ }
+ }
+
+ /**
+ * Returns all LogEntry in TxLog
+ */
+ public List<LogEntry> getLogEntries() {
+ return Collections.unmodifiableList(logEntries);
+ }
+
+ /**
+ * Returns true if TxLog is empty
+ */
+ public boolean isEmpty() {
+ return logEntries.isEmpty();
+ }
+
+ /**
+ * Returns a map of RowColumn changes given an operation
+ */
+ public Map<RowColumn, Bytes> getOperationMap(LogEntry.Operation op) {
+ Map<RowColumn, Bytes> opMap = new HashMap<>();
+ for (LogEntry entry : logEntries) {
+ if (entry.getOp().equals(op)) {
+ opMap.put(new RowColumn(entry.getRow(), entry.getColumn()), entry.getValue());
+ }
+ }
+ return opMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/Encoder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/Encoder.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/Encoder.java
new file mode 100644
index 0000000..f437b16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/Encoder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.data.Bytes;
+
+/**
+ * Transforms Java primitives to and from bytes using desired encoding
+ *
+ * @since 1.0.0
+ */
+public interface Encoder {
+
+ /**
+ * Encodes an integer to {@link Bytes}
+ */
+ Bytes encode(int i);
+
+ /**
+ * Encodes a long to {@link Bytes}
+ */
+ Bytes encode(long l);
+
+ /**
+ * Encodes a String to {@link Bytes}
+ */
+ Bytes encode(String s);
+
+ /**
+ * Encodes a float to {@link Bytes}
+ */
+ Bytes encode(float f);
+
+ /**
+ * Encodes a double to {@link Bytes}
+ */
+ Bytes encode(double d);
+
+ /**
+ * Encodes a boolean to {@link Bytes}
+ */
+ Bytes encode(boolean b);
+
+ /**
+ * Decodes an integer from {@link Bytes}
+ */
+ int decodeInteger(Bytes b);
+
+ /**
+ * Decodes a long from {@link Bytes}
+ */
+ long decodeLong(Bytes b);
+
+ /**
+ * Decodes a String from {@link Bytes}
+ */
+ String decodeString(Bytes b);
+
+ /**
+ * Decodes a float from {@link Bytes}
+ */
+ float decodeFloat(Bytes b);
+
+ /**
+ * Decodes a double from {@link Bytes}
+ */
+ double decodeDouble(Bytes b);
+
+ /**
+ * Decodes a boolean from {@link Bytes}
+ */
+ boolean decodeBoolean(Bytes b);
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/StringEncoder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/StringEncoder.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/StringEncoder.java
new file mode 100644
index 0000000..939f97a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/StringEncoder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.data.Bytes;
+
+/**
+ * Transforms Java primitives to and from bytes using a String encoding
+ *
+ * @since 1.0.0
+ */
+public class StringEncoder implements Encoder {
+
+ @Override
+ public Bytes encode(int i) {
+ return encode(Integer.toString(i));
+ }
+
+ @Override
+ public Bytes encode(long l) {
+ return encode(Long.toString(l));
+ }
+
+ @Override
+ public Bytes encode(String s) {
+ return Bytes.of(s);
+ }
+
+ @Override
+ public Bytes encode(float f) {
+ return encode(Float.toString(f));
+ }
+
+ @Override
+ public Bytes encode(double d) {
+ return encode(Double.toString(d));
+ }
+
+ @Override
+ public Bytes encode(boolean b) {
+ return encode(Boolean.toString(b));
+ }
+
+ @Override
+ public int decodeInteger(Bytes b) {
+ return Integer.parseInt(decodeString(b));
+ }
+
+ @Override
+ public long decodeLong(Bytes b) {
+ return Long.parseLong(decodeString(b));
+ }
+
+ @Override
+ public String decodeString(Bytes b) {
+ return b.toString();
+ }
+
+ @Override
+ public float decodeFloat(Bytes b) {
+ return Float.parseFloat(decodeString(b));
+ }
+
+ @Override
+ public double decodeDouble(Bytes b) {
+ return Double.parseDouble(decodeString(b));
+ }
+
+ @Override
+ public boolean decodeBoolean(Bytes b) {
+ return Boolean.parseBoolean(decodeString(b));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypeLayer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypeLayer.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypeLayer.java
new file mode 100644
index 0000000..82639f4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypeLayer.java
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import java.nio.ByteBuffer;
+
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+
+/**
+ * A simple convenience layer for Fluo. This layer attempts to make the following common operations
+ * easier.
+ *
+ * <UL>
+ * <LI>Working with different types.
+ * <LI>Supplying default values
+ * <LI>Dealing with null return types.
+ * <LI>Working with row/column and column maps
+ * </UL>
+ *
+ * <p>
+ * This layer was intentionally loosely coupled with the basic API. This allows other convenience
+ * layers for Fluo to build directly on the basic API w/o having to consider the particulars of this
+ * layer. Also its expected that integration with other languages may only use the basic API.
+ * </p>
+ *
+ * <h3>Using</h3>
+ *
+ * <p>
+ * A TypeLayer is created with a certain encoder that is used for converting from bytes to
+ * primitives and visa versa. In order to ensure that all of your code uses the same encoder, its
+ * probably best to centralize the choice of an encoder within your project. There are many ways do
+ * to this, below is an example of one way to centralize and use.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * public class MyTypeLayer extends TypeLayer {
+ * public MyTypeLayer() {
+ * super(new MyEncoder());
+ * }
+ * }
+ *
+ * public class MyObserver extends TypedObserver {
+ * MyObserver(){
+ * super(new MyTypeLayer());
+ * }
+ *
+ * public abstract void process(TypedTransaction tx, Bytes row, Column col){
+ * //do something w/ typed transaction
+ * }
+ * }
+ *
+ * public class MyUtil {
+ * //A little util to print out some stuff
+ * public void printStuff(Snapshot snap, byte[] row){
+ * TypedSnapshot tsnap = new MytTypeLayer().wrap(snap);
+ *
+ * System.out.println(tsnap.get().row(row).fam("b90000").qual(137).toString("NP"));
+ * }
+ * }
+ * </code>
+ * </pre>
+ *
+ * <h3>Working with different types</h3>
+ *
+ * <p>
+ * The following example code shows using the basic fluo API with different types.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void process(Transaction tx, byte[] row, byte[] cf, int cq, long val){
+ * tx.set(Bytes.of(row), new Column(Bytes.of(cf), Bytes.of(Integer.toString(cq))),
+ * Bytes.of(Long.toString(val));
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * Alternatively, the same thing can be written using a {@link TypedTransactionBase} in the
+ * following way. Because row(), fam(), qual(), and set() each take many different types, this
+ * enables many different permutations that would not be achievable with overloading.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void process(TypedTransaction tx, byte[] r, byte[] cf, int cq, long v){
+ * tx.mutate().row(r).fam(cf).qual(cq).set(v);
+ * }
+ * </code>
+ * </pre>
+ *
+ * <h3>Default values</h3>
+ *
+ * <p>
+ * The following example code shows using the basic fluo API to read a value and default to zero if
+ * it does not exist.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void add(Transaction tx, byte[] row, Column col, long amount){
+ *
+ * long balance = 0;
+ * Bytes bval = tx.get(Bytes.of(row), col);
+ * if(bval != null)
+ * balance = Long.parseLong(bval.toString());
+ *
+ * balance += amount;
+ *
+ * tx.set(Bytes.of(row), col, Bytes.of(Long.toString(amount)));
+ *
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * Alternatively, the same thing can be written using a {@link TypedTransactionBase} in the
+ * following way. This code avoids the null check by supplying a default value of zero.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void add(TypedTransaction tx, byte[] r, Column c, long amount){
+ * long balance = tx.get().row(r).col(c).toLong(0);
+ * balance += amount;
+ * tx.mutate().row(r).col(c).set(balance);
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * For this particular case, shorter code can be written by using the increment method.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void add(TypedTransaction tx, byte[] r, Column c, long amount){
+ * tx.mutate().row(r).col(c).increment(amount);
+ * }
+ * </code>
+ * </pre>
+ *
+ * <h3>Null return types</h3>
+ *
+ * <p>
+ * When using the basic API, you must ensure the return type is not null before converting a string
+ * or long.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void process(Transaction tx, byte[] row, Column col, long amount) {
+ * Bytes val = tx.get(Bytes.of(row), col);
+ * if(val == null)
+ * return;
+ * long balance = Long.parseLong(val.toString());
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * With {@link TypedTransactionBase} if no default value is supplied, then the null is passed
+ * through.
+ * </p>
+ *
+ * <pre>
+ * <code>
+ *
+ * void process(TypedTransaction tx, byte[] r, Column c, long amount){
+ * Long balance = tx.get().row(r).col(c).toLong();
+ * if(balance == null)
+ * return;
+ * }
+ * </code>
+ * </pre>
+ *
+ * <h3>Defaulted maps</h3>
+ *
+ * <p>
+ * The operations that return maps, return defaulted maps which make it easy to specify defaults and
+ * avoid null.
+ * </p>
+ *
+ * <pre>
+ * {@code
+ * // pretend this method has curly braces. javadoc has issues with less than.
+ *
+ * void process(TypedTransaction tx, byte[] r, Column c1, Column c2, Column c3, long amount)
+ *
+ * Map<Column, Value> columns = tx.get().row(r).columns(c1,c2,c3);
+ *
+ * // If c1 does not exist in map, a Value that wraps null will be returned.
+ * // When c1 does not exist val1 will be set to null and no NPE will be thrown.
+ * String val1 = columns.get(c1).toString();
+ *
+ * // If c2 does not exist in map, then val2 will be set to empty string.
+ * String val2 = columns.get(c2).toString("");
+ *
+ * // If c3 does not exist in map, then val9 will be set to 9.
+ * Long val3 = columns.get(c3).toLong(9);
+ * }
+ * </pre>
+ *
+ * <p>
+ * This also applies to getting sets of rows.
+ * </p>
+ *
+ * <pre>
+ * {@code
+ * // pretend this method has curly braces. javadoc has issues with less than.
+ *
+ * void process(TypedTransaction tx, List<String> rows, Column c1, Column c2, Column c3,
+ * long amount)
+ *
+ * Map<String,Map<Column,Value>> rowCols =
+ * tx.get().rowsString(rows).columns(c1,c2,c3).toStringMap();
+ *
+ * // this will set val1 to null if row does not exist in map and/or column does not
+ * // exist in child map
+ * String val1 = rowCols.get("row1").get(c1).toString();
+ * }
+ * </pre>
+ *
+ * @since 1.0.0
+ */
+public class TypeLayer {
+
+ private Encoder encoder;
+
+ static class Data {
+ Bytes row;
+ Bytes family;
+ Bytes qual;
+ Bytes vis;
+
+ Column getCol() {
+ if (qual == null) {
+ return new Column(family);
+ } else if (vis == null) {
+ return new Column(family, qual);
+ } else {
+ return new Column(family, qual, vis);
+ }
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public abstract class RowMethods<R> {
+
+ abstract R create(Data data);
+
+ public R row(String row) {
+ return row(encoder.encode(row));
+ }
+
+ public R row(int row) {
+ return row(encoder.encode(row));
+ }
+
+ public R row(long row) {
+ return row(encoder.encode(row));
+ }
+
+ public R row(byte[] row) {
+ return row(Bytes.of(row));
+ }
+
+ public R row(ByteBuffer row) {
+ return row(Bytes.of(row));
+ }
+
+ public R row(Bytes row) {
+ Data data = new Data();
+ data.row = row;
+ R result = create(data);
+ return result;
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public abstract class SimpleFamilyMethods<R1> {
+
+ Data data;
+
+ SimpleFamilyMethods(Data data) {
+ this.data = data;
+ }
+
+ abstract R1 create1(Data data);
+
+ public R1 fam(String family) {
+ return fam(encoder.encode(family));
+ }
+
+ public R1 fam(int family) {
+ return fam(encoder.encode(family));
+ }
+
+ public R1 fam(long family) {
+ return fam(encoder.encode(family));
+ }
+
+ public R1 fam(byte[] family) {
+ return fam(Bytes.of(family));
+ }
+
+ public R1 fam(ByteBuffer family) {
+ return fam(Bytes.of(family));
+ }
+
+ public R1 fam(Bytes family) {
+ data.family = family;
+ return create1(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public abstract class FamilyMethods<R1, R2> extends SimpleFamilyMethods<R1> {
+
+ FamilyMethods(Data data) {
+ super(data);
+ }
+
+ abstract R2 create2(Data data);
+
+ public R2 col(Column col) {
+ data.family = col.getFamily();
+ data.qual = col.getQualifier();
+ data.vis = col.getVisibility();
+ return create2(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public abstract class QualifierMethods<R> {
+
+ private Data data;
+
+ QualifierMethods(Data data) {
+ this.data = data;
+ }
+
+ abstract R create(Data data);
+
+ public R qual(String qualifier) {
+ return qual(encoder.encode(qualifier));
+ }
+
+ public R qual(int qualifier) {
+ return qual(encoder.encode(qualifier));
+ }
+
+ public R qual(long qualifier) {
+ return qual(encoder.encode(qualifier));
+ }
+
+ public R qual(byte[] qualifier) {
+ return qual(Bytes.of(qualifier));
+ }
+
+ public R qual(ByteBuffer qualifier) {
+ return qual(Bytes.of(qualifier));
+ }
+
+ public R qual(Bytes qualifier) {
+ data.qual = qualifier;
+ return create(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public static class VisibilityMethods {
+
+ private Data data;
+
+ VisibilityMethods(Data data) {
+ this.data = data;
+ }
+
+ public Column vis() {
+ return new Column(data.family, data.qual);
+ }
+
+ public Column vis(String cv) {
+ return vis(Bytes.of(cv));
+ }
+
+ public Column vis(Bytes cv) {
+ return new Column(data.family, data.qual, cv);
+ }
+
+ public Column vis(ByteBuffer cv) {
+ return vis(Bytes.of(cv));
+ }
+
+ public Column vis(byte[] cv) {
+ return vis(Bytes.of(cv));
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class CQB extends QualifierMethods<VisibilityMethods> {
+ CQB(Data data) {
+ super(data);
+ }
+
+ @Override
+ VisibilityMethods create(Data data) {
+ return new VisibilityMethods(data);
+ }
+ }
+
+ /**
+ * @since 1.0.0
+ */
+ public class CFB extends SimpleFamilyMethods<CQB> {
+ CFB() {
+ super(new Data());
+ }
+
+ @Override
+ CQB create1(Data data) {
+ return new CQB(data);
+ }
+ }
+
+ public TypeLayer(Encoder encoder) {
+ this.encoder = encoder;
+ }
+
+ /**
+ * Initiates the chain of calls needed to build a column.
+ *
+ * @return a column builder
+ */
+ public CFB bc() {
+ return new CFB();
+ }
+
+ public TypedSnapshot wrap(Snapshot snap) {
+ return new TypedSnapshot(snap, encoder, this);
+ }
+
+ public TypedTransactionBase wrap(TransactionBase tx) {
+ return new TypedTransactionBase(tx, encoder, this);
+ }
+
+ public TypedTransaction wrap(Transaction tx) {
+ return new TypedTransaction(tx, encoder, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedLoader.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedLoader.java
new file mode 100644
index 0000000..0e14062
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedLoader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.client.Loader;
+import org.apache.fluo.api.client.TransactionBase;
+
+/**
+ * A {@link Loader} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public abstract class TypedLoader implements Loader {
+
+ private final TypeLayer tl;
+
+ public TypedLoader() {
+ tl = new TypeLayer(new StringEncoder());
+ }
+
+ public TypedLoader(TypeLayer tl) {
+ this.tl = tl;
+ }
+
+ @Override
+ public void load(TransactionBase tx, Context context) throws Exception {
+ load(tl.wrap(tx), context);
+ }
+
+ public abstract void load(TypedTransactionBase tx, Context context) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedObserver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedObserver.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedObserver.java
new file mode 100644
index 0000000..ca68285
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedObserver.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.observer.AbstractObserver;
+
+/**
+ * An {@link AbstractObserver} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public abstract class TypedObserver extends AbstractObserver {
+
+ private final TypeLayer tl;
+
+ public TypedObserver() {
+ tl = new TypeLayer(new StringEncoder());
+ }
+
+ public TypedObserver(TypeLayer tl) {
+ this.tl = tl;
+ }
+
+ @Override
+ public void process(TransactionBase tx, Bytes row, Column col) {
+ process(tl.wrap(tx), row, col);
+ }
+
+ public abstract void process(TypedTransactionBase tx, Bytes row, Column col);
+}
http://git-wip-us.apache.org/repos/asf/incubator-fluo-recipes/blob/beea3f96/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshot.java b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshot.java
new file mode 100644
index 0000000..6c09764
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/fluo/recipes/core/types/TypedSnapshot.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.fluo.recipes.core.types;
+
+import org.apache.fluo.api.client.Snapshot;
+
+/**
+ * A {@link Snapshot} that uses a {@link TypeLayer}
+ *
+ * @since 1.0.0
+ */
+public class TypedSnapshot extends TypedSnapshotBase implements Snapshot {
+
+ private final Snapshot closeSnapshot;
+
+ TypedSnapshot(Snapshot snapshot, Encoder encoder, TypeLayer tl) {
+ super(snapshot, encoder, tl);
+ closeSnapshot = snapshot;
+ }
+
+ @Override
+ public void close() {
+ closeSnapshot.close();
+ }
+}