You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2017/09/13 12:13:15 UTC
[5/5] incubator-hivemall git commit: Close #114: refactored
to_ordered_(map|list) of PR #108
Close #114: refactored to_ordered_(map|list) of PR #108
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/688daa5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/688daa5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/688daa5f
Branch: refs/heads/master
Commit: 688daa5f8e6a87fad2abf3b47a8a8353bc1c792a
Parents: 69730f6
Author: Makoto Yui <my...@apache.org>
Authored: Wed Sep 13 21:12:41 2017 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Wed Sep 13 21:12:41 2017 +0900
----------------------------------------------------------------------
.../hivemall/tools/map/UDAFToOrderedMap.java | 240 ++++++++++++++++---
.../java/hivemall/utils/hadoop/HiveUtils.java | 12 +
.../tools/map/UDAFToOrderedMapTest.java | 36 +--
docs/gitbook/misc/generic_funcs.md | 54 ++++-
4 files changed, 292 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/688daa5f/core/src/main/java/hivemall/tools/map/UDAFToOrderedMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/UDAFToOrderedMap.java b/core/src/main/java/hivemall/tools/map/UDAFToOrderedMap.java
index a6b547f..5cdac4d 100644
--- a/core/src/main/java/hivemall/tools/map/UDAFToOrderedMap.java
+++ b/core/src/main/java/hivemall/tools/map/UDAFToOrderedMap.java
@@ -20,11 +20,17 @@ package hivemall.tools.map;
import hivemall.utils.collections.maps.BoundedSortedMap;
import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.Preconditions;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
+import java.util.Map;
import java.util.TreeMap;
import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -33,8 +39,16 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.IntWritable;
/**
* Convert two aggregated columns into a sorted key-value map.
@@ -63,70 +77,232 @@ public final class UDAFToOrderedMap extends UDAFToMap {
int size = 0;
if (typeInfo.length == 3) {
ObjectInspector[] argOIs = info.getParameterObjectInspectors();
- if (HiveUtils.isBooleanTypeInfo(typeInfo[2])) {
- reverseOrder = HiveUtils.getConstBoolean(argOIs[2]);
- } else if (HiveUtils.isIntegerTypeInfo(typeInfo[2])) {
- size = HiveUtils.getConstInt(argOIs[2]);
+ ObjectInspector argOI2 = argOIs[2];
+ if (HiveUtils.isConstBoolean(argOI2)) {
+ reverseOrder = HiveUtils.getConstBoolean(argOI2);
+ } else if (HiveUtils.isConstInteger(argOI2)) {
+ size = HiveUtils.getConstInt(argOI2);
if (size == 0) {
- throw new UDFArgumentException("Map size must be nonzero: " + size);
+ throw new UDFArgumentException("Map size must be non-zero value: " + size);
}
reverseOrder = (size > 0); // positive size => top-k
- size = Math.abs(size);
} else {
throw new UDFArgumentTypeException(2,
- "The third argument must be boolean or integer type: "
- + typeInfo[2].getTypeName());
+ "The third argument must be boolean or int type: " + typeInfo[2].getTypeName());
}
}
if (reverseOrder) { // descending
- return new DescendingMapEvaluator(size);
+ if (size == 0) {
+ return new ReverseOrderedMapEvaluator();
+ } else {
+ return new TopKOrderedMapEvaluator();
+ }
} else { // ascending
- return new AscendingMapEvaluator(size);
+ if (size == 0) {
+ return new NaturalOrderedMapEvaluator();
+ } else {
+ return new TailKOrderedMapEvaluator();
+ }
}
}
- public static final class AscendingMapEvaluator extends UDAFToMapEvaluator {
-
- private final int size;
+ public static class NaturalOrderedMapEvaluator extends UDAFToMapEvaluator {
- AscendingMapEvaluator(@Nonnegative int size) {
- super();
- this.size = size;
+ @Override
+ public void reset(@SuppressWarnings("deprecation") AggregationBuffer agg)
+ throws HiveException {
+ ((MapAggregationBuffer) agg).container = new TreeMap<Object, Object>();
}
+ }
+
+ public static class ReverseOrderedMapEvaluator extends UDAFToMapEvaluator {
+
@Override
public void reset(@SuppressWarnings("deprecation") AggregationBuffer agg)
throws HiveException {
- if (size == 0) {
- ((MapAggregationBuffer) agg).container = new TreeMap<Object, Object>();
- } else {
- ((MapAggregationBuffer) agg).container = new BoundedSortedMap<Object, Object>(size);
- }
+ ((MapAggregationBuffer) agg).container = new TreeMap<Object, Object>(
+ Collections.reverseOrder());
}
}
- public static final class DescendingMapEvaluator extends UDAFToMapEvaluator {
+ public static class TopKOrderedMapEvaluator extends GenericUDAFEvaluator {
+
+ protected PrimitiveObjectInspector inputKeyOI;
+ protected ObjectInspector inputValueOI;
+ protected MapObjectInspector partialMapOI;
+ protected PrimitiveObjectInspector sizeOI;
+
+ protected StructObjectInspector internalMergeOI;
+
+ protected StructField partialMapField;
+ protected StructField sizeField;
- private final int size;
+ @Override
+ public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
+ super.init(mode, argOIs);
+
+ // initialize input
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
+ this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
+ this.inputValueOI = argOIs[1];
+ this.sizeOI = HiveUtils.asIntegerOI(argOIs[2]);
+ } else {// from partial aggregation
+ StructObjectInspector soi = (StructObjectInspector) argOIs[0];
+ this.internalMergeOI = soi;
+
+ this.partialMapField = soi.getStructFieldRef("partialMap");
+ // re-extract input key/value OIs
+ MapObjectInspector partialMapOI = (MapObjectInspector) partialMapField.getFieldObjectInspector();
+ this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(partialMapOI.getMapKeyObjectInspector());
+ this.inputValueOI = partialMapOI.getMapValueObjectInspector();
+
+ this.partialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
+ ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
+ ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
+
+ this.sizeField = soi.getStructFieldRef("size");
+ this.sizeOI = (PrimitiveObjectInspector) sizeField.getFieldObjectInspector();
+ }
- DescendingMapEvaluator(int size) {
- super();
- this.size = size;
+ // initialize output
+ final ObjectInspector outputOI;
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
+ outputOI = internalMergeOI(inputKeyOI, inputValueOI);
+ } else {// terminate
+ outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
+ ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
+ ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
+ }
+ return outputOI;
+ }
+
+ @Nonnull
+ private static StructObjectInspector internalMergeOI(
+ @Nonnull PrimitiveObjectInspector keyOI, @Nonnull ObjectInspector valueOI) {
+ List<String> fieldNames = new ArrayList<String>();
+ List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+
+ fieldNames.add("partialMap");
+ fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
+ ObjectInspectorUtils.getStandardObjectInspector(keyOI),
+ ObjectInspectorUtils.getStandardObjectInspector(valueOI)));
+
+ fieldNames.add("size");
+ fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ static class MapAggregationBuffer extends AbstractAggregationBuffer {
+ @Nullable
+ Map<Object, Object> container;
+ int size;
+
+ MapAggregationBuffer() {
+ super();
+ }
}
@Override
public void reset(@SuppressWarnings("deprecation") AggregationBuffer agg)
throws HiveException {
- if (size == 0) {
- ((MapAggregationBuffer) agg).container = new TreeMap<Object, Object>(
- Collections.reverseOrder());
- } else {
- ((MapAggregationBuffer) agg).container = new BoundedSortedMap<Object, Object>(size,
- true);
+ MapAggregationBuffer myagg = (MapAggregationBuffer) agg;
+ myagg.container = null;
+ myagg.size = 0;
+ }
+
+ @Override
+ public MapAggregationBuffer getNewAggregationBuffer() throws HiveException {
+ MapAggregationBuffer myagg = new MapAggregationBuffer();
+ reset(myagg);
+ return myagg;
+ }
+
+ @Override
+ public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
+ Object[] parameters) throws HiveException {
+ assert (parameters.length == 3);
+ if (parameters[0] == null) {
+ return;
+ }
+
+ Object key = ObjectInspectorUtils.copyToStandardObject(parameters[0], inputKeyOI);
+ Object value = ObjectInspectorUtils.copyToStandardObject(parameters[1], inputValueOI);
+ int size = Math.abs(HiveUtils.getInt(parameters[2], sizeOI)); // size could be negative for tail-k
+
+ MapAggregationBuffer myagg = (MapAggregationBuffer) agg;
+ if (myagg.container == null) {
+ initBuffer(myagg, size);
+ }
+ myagg.container.put(key, value);
+ }
+
+ void initBuffer(@Nonnull MapAggregationBuffer agg, @Nonnegative int size) {
+ Preconditions.checkArgument(size > 0, "size MUST be greather than zero: " + size);
+
+ agg.container = new BoundedSortedMap<Object, Object>(size, true);
+ agg.size = size;
+ }
+
+ @Override
+ public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
+ throws HiveException {
+ MapAggregationBuffer myagg = (MapAggregationBuffer) agg;
+
+ Object[] partialResult = new Object[2];
+ partialResult[0] = myagg.container;
+ partialResult[1] = new IntWritable(myagg.size);
+
+ return partialResult;
+ }
+
+ @Override
+ public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
+ throws HiveException {
+ if (partial == null) {
+ return;
+ }
+
+ MapAggregationBuffer myagg = (MapAggregationBuffer) agg;
+
+ Object partialMapObj = internalMergeOI.getStructFieldData(partial, partialMapField);
+ Map<?, ?> partialMap = partialMapOI.getMap(HiveUtils.castLazyBinaryObject(partialMapObj));
+ if (partialMap == null) {
+ return;
+ }
+
+ if (myagg.container == null) {
+ Object sizeObj = internalMergeOI.getStructFieldData(partial, sizeField);
+ int size = HiveUtils.getInt(sizeObj, sizeOI);
+ initBuffer(myagg, size);
+ }
+ for (Map.Entry<?, ?> e : partialMap.entrySet()) {
+ Object key = ObjectInspectorUtils.copyToStandardObject(e.getKey(), inputKeyOI);
+ Object value = ObjectInspectorUtils.copyToStandardObject(e.getValue(), inputValueOI);
+ myagg.container.put(key, value);
}
}
+ @Override
+ @Nullable
+ public Map<Object, Object> terminate(@SuppressWarnings("deprecation") AggregationBuffer agg)
+ throws HiveException {
+ MapAggregationBuffer myagg = (MapAggregationBuffer) agg;
+ return myagg.container;
+ }
+
}
+
+ public static class TailKOrderedMapEvaluator extends TopKOrderedMapEvaluator {
+
+ @Override
+ void initBuffer(MapAggregationBuffer agg, int size) {
+ agg.container = new BoundedSortedMap<Object, Object>(size);
+ agg.size = size;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/688daa5f/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
index afa8a58..8fba349 100644
--- a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
+++ b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
@@ -326,6 +326,18 @@ public final class HiveUtils {
return ObjectInspectorUtils.isConstantObjectInspector(oi) && isStringOI(oi);
}
+ public static boolean isConstInt(@Nonnull final ObjectInspector oi) {
+ return ObjectInspectorUtils.isConstantObjectInspector(oi) && isIntOI(oi);
+ }
+
+ public static boolean isConstInteger(@Nonnull final ObjectInspector oi) {
+ return ObjectInspectorUtils.isConstantObjectInspector(oi) && isIntegerOI(oi);
+ }
+
+ public static boolean isConstBoolean(@Nonnull final ObjectInspector oi) {
+ return ObjectInspectorUtils.isConstantObjectInspector(oi) && isBooleanOI(oi);
+ }
+
public static boolean isPrimitiveTypeInfo(@Nonnull TypeInfo typeInfo) {
return typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE;
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/688daa5f/core/src/test/java/hivemall/tools/map/UDAFToOrderedMapTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/map/UDAFToOrderedMapTest.java b/core/src/test/java/hivemall/tools/map/UDAFToOrderedMapTest.java
index 61642f1..38bc5ae 100644
--- a/core/src/test/java/hivemall/tools/map/UDAFToOrderedMapTest.java
+++ b/core/src/test/java/hivemall/tools/map/UDAFToOrderedMapTest.java
@@ -18,10 +18,12 @@
*/
package hivemall.tools.map;
-import hivemall.tools.map.UDAFToOrderedMap.AscendingMapEvaluator;
-import hivemall.tools.map.UDAFToOrderedMap.DescendingMapEvaluator;
+import hivemall.tools.map.UDAFToOrderedMap.NaturalOrderedMapEvaluator;
+import hivemall.tools.map.UDAFToOrderedMap.ReverseOrderedMapEvaluator;
+import hivemall.tools.map.UDAFToOrderedMap.TailKOrderedMapEvaluator;
+import hivemall.tools.map.UDAFToOrderedMap.TopKOrderedMapEvaluator;
-import java.util.SortedMap;
+import java.util.Map;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -33,8 +35,8 @@ public class UDAFToOrderedMapTest {
@Test
public void testNaturalOrder() throws Exception {
- AscendingMapEvaluator evaluator = new AscendingMapEvaluator(0);
- AscendingMapEvaluator.MapAggregationBuffer agg = (AscendingMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
+ NaturalOrderedMapEvaluator evaluator = new NaturalOrderedMapEvaluator();
+ NaturalOrderedMapEvaluator.MapAggregationBuffer agg = (NaturalOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
ObjectInspector[] inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
@@ -50,7 +52,7 @@ public class UDAFToOrderedMapTest {
evaluator.iterate(agg, new Object[] {keys[i], values[i]});
}
- SortedMap<Object, Object> res = (SortedMap<Object, Object>) evaluator.terminate(agg);
+ Map<Object, Object> res = evaluator.terminate(agg);
Object[] sortedValues = res.values().toArray();
Assert.assertEquals(3, sortedValues.length);
@@ -63,8 +65,8 @@ public class UDAFToOrderedMapTest {
@Test
public void testReverseOrder() throws Exception {
- DescendingMapEvaluator evaluator = new DescendingMapEvaluator(0);
- DescendingMapEvaluator.MapAggregationBuffer agg = (DescendingMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
+ ReverseOrderedMapEvaluator evaluator = new ReverseOrderedMapEvaluator();
+ ReverseOrderedMapEvaluator.MapAggregationBuffer agg = (ReverseOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
ObjectInspector[] inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
@@ -81,7 +83,7 @@ public class UDAFToOrderedMapTest {
evaluator.iterate(agg, new Object[] {keys[i], values[i]});
}
- SortedMap<Object, Object> res = (SortedMap<Object, Object>) evaluator.terminate(agg);
+ Map<Object, Object> res = evaluator.terminate(agg);
Object[] sortedValues = res.values().toArray();
Assert.assertEquals(3, sortedValues.length);
@@ -94,9 +96,8 @@ public class UDAFToOrderedMapTest {
@Test
public void testTopK() throws Exception {
- int size = 2;
- DescendingMapEvaluator evaluator = new DescendingMapEvaluator(size);
- DescendingMapEvaluator.MapAggregationBuffer agg = (DescendingMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
+ TopKOrderedMapEvaluator evaluator = new TopKOrderedMapEvaluator();
+ TopKOrderedMapEvaluator.MapAggregationBuffer agg = (TopKOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
ObjectInspector[] inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
@@ -105,6 +106,7 @@ public class UDAFToOrderedMapTest {
final double[] keys = new double[] {0.7, 0.5, 0.8};
final String[] values = new String[] {"banana", "apple", "candy"};
+ int size = 2;
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
@@ -113,7 +115,7 @@ public class UDAFToOrderedMapTest {
evaluator.iterate(agg, new Object[] {keys[i], values[i], size});
}
- SortedMap<Object, Object> res = (SortedMap<Object, Object>) evaluator.terminate(agg);
+ Map<Object, Object> res = evaluator.terminate(agg);
Object[] sortedValues = res.values().toArray();
Assert.assertEquals(size, sortedValues.length);
@@ -125,9 +127,8 @@ public class UDAFToOrderedMapTest {
@Test
public void testTailK() throws Exception {
- int size = -2;
- AscendingMapEvaluator evaluator = new AscendingMapEvaluator(Math.abs(size));
- AscendingMapEvaluator.MapAggregationBuffer agg = (AscendingMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
+ TailKOrderedMapEvaluator evaluator = new TailKOrderedMapEvaluator();
+ TailKOrderedMapEvaluator.MapAggregationBuffer agg = (TailKOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();
ObjectInspector[] inputOIs = new ObjectInspector[] {
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
@@ -136,6 +137,7 @@ public class UDAFToOrderedMapTest {
final double[] keys = new double[] {0.7, 0.5, 0.8};
final String[] values = new String[] {"banana", "apple", "candy"};
+ int size = -2;
evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
evaluator.reset(agg);
@@ -144,7 +146,7 @@ public class UDAFToOrderedMapTest {
evaluator.iterate(agg, new Object[] {keys[i], values[i], size});
}
- SortedMap<Object, Object> res = (SortedMap<Object, Object>) evaluator.terminate(agg);
+ Map<Object, Object> res = evaluator.terminate(agg);
Object[] sortedValues = res.values().toArray();
Assert.assertEquals(Math.abs(size), sortedValues.length);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/688daa5f/docs/gitbook/misc/generic_funcs.md
----------------------------------------------------------------------
diff --git a/docs/gitbook/misc/generic_funcs.md b/docs/gitbook/misc/generic_funcs.md
index 66b30e2..03e1ef3 100644
--- a/docs/gitbook/misc/generic_funcs.md
+++ b/docs/gitbook/misc/generic_funcs.md
@@ -85,7 +85,36 @@ This page describes a list of useful Hivemall generic functions.
## List UDAF
-- `to_ordered_list(value [, const string options])` or `to_ordered_list(value, key [, const string options])` - Return list of values sorted by value itself or specific key
+- `to_ordered_list(PRIMITIVE value [, PRIMITIVE key, const string options])` or `to_ordered_list(value, key [, const string options])` - Return list of values sorted by value itself or specific key
+
+ ```sql
+ with t as (
+ select 5 as key, 'apple' as value
+ union all
+ select 3 as key, 'banana' as value
+ union all
+ select 4 as key, 'candy' as value
+ union all
+ select 2 as key, 'donut' as value
+ union all
+ select 3 as key, 'egg' as value
+ )
+ select -- expected output
+ to_ordered_list(value, key, '-reverse'), -- [apple, candy, (banana, egg | egg, banana), donut] (reverse order)
+ to_ordered_list(value, key, '-k 2'), -- [apple, candy] (top-k)
+ to_ordered_list(value, key, '-k 100'), -- [apple, candy, (banana, egg | egg, banana), dunut]
+ to_ordered_list(value, key, '-k 2 -reverse'), -- [donut, (banana | egg)] (reverse top-k = tail-k)
+ to_ordered_list(value, key), -- [donut, (banana, egg | egg, banana), candy, apple] (natural order)
+ to_ordered_list(value, key, '-k -2'), -- [donut, (banana | egg)] (tail-k)
+ to_ordered_list(value, key, '-k -100'), -- [donut, (banana, egg | egg, banana), candy, apple]
+ to_ordered_list(value, key, '-k -2 -reverse'), -- [apple, candy] (reverse tail-k = top-k)
+ to_ordered_list(value, '-k 2'), -- [egg, donut] (alphabetically)
+ to_ordered_list(key, '-k -2 -reverse'), -- [5, 4] (top-2 keys)
+ to_ordered_list(key) -- [2, 3, 3, 4, 5] (natural ordered keys)
+ from
+ t
+ ;
+ ```
# Bitset functions
@@ -147,6 +176,29 @@ The compression level must be in range [-1,9]
- `to_ordered_map(key, value [, const int k|const boolean reverseOrder=false])` - Convert two aggregated columns into an ordered key-value map
+ ```sql
+ with t as (
+ select 10 as key, 'apple' as value
+ union all
+ select 3 as key, 'banana' as value
+ union all
+ select 4 as key, 'candy' as value
+ )
+ select
+ to_ordered_map(key, value, true), -- {10:"apple",4:"candy",3:"banana"} (reverse)
+ to_ordered_map(key, value, 1), -- {10:"apple"} (top-1)
+ to_ordered_map(key, value, 2), -- {10:"apple",4:"candy"} (top-2)
+ to_ordered_map(key, value, 3), -- {10:"apple",4:"candy",3:"banana"} (top-3)
+ to_ordered_map(key, value, 100), -- {10:"apple",4:"candy",3:"banana"} (top-100)
+ to_ordered_map(key, value), -- {3:"banana",4:"candy",10:"apple"} (natural)
+ to_ordered_map(key, value, -1), -- {3:"banana"} (tail-1)
+ to_ordered_map(key, value, -2), -- {3:"banana",4:"candy"} (tail-2)
+ to_ordered_map(key, value, -3), -- {3:"banana",4:"candy",10:"apple"} (tail-3)
+ to_ordered_map(key, value, -100) -- {3:"banana",4:"candy",10:"apple"} (tail-100)
+ from t
+ ;
+ ```
+
# MapReduce functions
- `rowid()` - Returns a generated row id of a form {TASK_ID}-{SEQUENCE_NUMBER}