You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2015/06/11 20:30:51 UTC
[1/2] drill git commit: DRILL-3200: Add Window functions: ROW_NUMBER,
RANK, PERCENT_RANK, DENSE_RANK and CUME_DIST
Repository: drill
Updated Branches:
refs/heads/master 453f6f7a8 -> 3bccec911
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p1/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p1/0.data.json b/exec/java-exec/src/test/resources/window/b1.p1/0.data.json
new file mode 100644
index 0000000..cd34641
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p1/0.data.json
@@ -0,0 +1,21 @@
+// partition rows 20, subs [1, 2, 3, 4, 5, 6]
+{ "employee_id":8, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":7, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":14, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":18, "position_id":1, "sub":6, "salary":16 }
+{ "employee_id":4, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":5, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":16, "position_id":1, "sub":6, "salary":16 }
+{ "employee_id":19, "position_id":1, "sub":6, "salary":16 }
+{ "employee_id":0, "position_id":1, "sub":1, "salary":11 }
+{ "employee_id":17, "position_id":1, "sub":6, "salary":16 }
+{ "employee_id":12, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":2, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":11, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":6, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":13, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":9, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":15, "position_id":1, "sub":6, "salary":16 }
+{ "employee_id":10, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":3, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":1, "position_id":1, "sub":2, "salary":12 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p2.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p2.subs.tsv b/exec/java-exec/src/test/resources/window/b1.p2.subs.tsv
new file mode 100644
index 0000000..edc4b6a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p2.subs.tsv
@@ -0,0 +1,20 @@
+1 11 1 1 1 0.1 0.0
+3 35 2 2 2 0.3 0.1111111111111111
+3 35 3 2 2 0.3 0.1111111111111111
+6 74 4 4 3 0.6 0.3333333333333333
+6 74 5 4 3 0.6 0.3333333333333333
+6 74 6 4 3 0.6 0.3333333333333333
+10 130 7 7 4 1.0 0.6666666666666666
+10 130 8 7 4 1.0 0.6666666666666666
+10 130 9 7 4 1.0 0.6666666666666666
+10 130 10 7 4 1.0 0.6666666666666666
+4 56 1 1 1 0.4 0.0
+4 56 2 1 1 0.4 0.0
+4 56 3 1 1 0.4 0.0
+4 56 4 1 1 0.4 0.0
+9 131 5 5 2 0.9 0.4444444444444444
+9 131 6 5 2 0.9 0.4444444444444444
+9 131 7 5 2 0.9 0.4444444444444444
+9 131 8 5 2 0.9 0.4444444444444444
+9 131 9 5 2 0.9 0.4444444444444444
+10 147 10 10 3 1.0 1.0
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p2.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p2.tsv b/exec/java-exec/src/test/resources/window/b1.p2.tsv
new file mode 100644
index 0000000..2bd0010
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p2.tsv
@@ -0,0 +1,20 @@
+10 130
+10 130
+10 130
+10 130
+10 130
+10 130
+10 130
+10 130
+10 130
+10 130
+10 147
+10 147
+10 147
+10 147
+10 147
+10 147
+10 147
+10 147
+10 147
+10 147
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p2/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p2/0.data.json b/exec/java-exec/src/test/resources/window/b1.p2/0.data.json
new file mode 100644
index 0000000..b70cf91
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p2/0.data.json
@@ -0,0 +1,22 @@
+// partition rows 10, subs [1, 2, 3, 4]
+// partition rows 10, subs [4, 5, 6]
+{ "employee_id":19, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":7, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":17, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":9, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":16, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":0, "position_id":1, "sub":1, "salary":11 }
+{ "employee_id":13, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":18, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":3, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":10, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":2, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":11, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":15, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":4, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":12, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":14, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":6, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":8, "position_id":1, "sub":4, "salary":14 }
+{ "employee_id":1, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":5, "position_id":1, "sub":3, "salary":13 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p2.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p2.subs.tsv b/exec/java-exec/src/test/resources/window/b2.p2.subs.tsv
new file mode 100644
index 0000000..153efaf
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p2.subs.tsv
@@ -0,0 +1,40 @@
+3 39 1 1 1 0.15 0.0
+3 39 2 1 1 0.15 0.0
+3 39 3 1 1 0.15 0.0
+8 114 4 4 2 0.4 0.15789473684210525
+8 114 5 4 2 0.4 0.15789473684210525
+8 114 6 4 2 0.4 0.15789473684210525
+8 114 7 4 2 0.4 0.15789473684210525
+8 114 8 4 2 0.4 0.15789473684210525
+20 342 9 9 3 1.0 0.42105263157894735
+20 342 10 9 3 1.0 0.42105263157894735
+20 342 11 9 3 1.0 0.42105263157894735
+20 342 12 9 3 1.0 0.42105263157894735
+20 342 13 9 3 1.0 0.42105263157894735
+20 342 14 9 3 1.0 0.42105263157894735
+20 342 15 9 3 1.0 0.42105263157894735
+20 342 16 9 3 1.0 0.42105263157894735
+20 342 17 9 3 1.0 0.42105263157894735
+20 342 18 9 3 1.0 0.42105263157894735
+20 342 19 9 3 1.0 0.42105263157894735
+20 342 20 9 3 1.0 0.42105263157894735
+9 171 1 1 1 0.45 0.0
+9 171 2 1 1 0.45 0.0
+9 171 3 1 1 0.45 0.0
+9 171 4 1 1 0.45 0.0
+9 171 5 1 1 0.45 0.0
+9 171 6 1 1 0.45 0.0
+9 171 7 1 1 0.45 0.0
+9 171 8 1 1 0.45 0.0
+9 171 9 1 1 0.45 0.0
+20 391 10 10 2 1.0 0.47368421052631576
+20 391 11 10 2 1.0 0.47368421052631576
+20 391 12 10 2 1.0 0.47368421052631576
+20 391 13 10 2 1.0 0.47368421052631576
+20 391 14 10 2 1.0 0.47368421052631576
+20 391 15 10 2 1.0 0.47368421052631576
+20 391 16 10 2 1.0 0.47368421052631576
+20 391 17 10 2 1.0 0.47368421052631576
+20 391 18 10 2 1.0 0.47368421052631576
+20 391 19 10 2 1.0 0.47368421052631576
+20 391 20 10 2 1.0 0.47368421052631576
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p2.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p2.tsv b/exec/java-exec/src/test/resources/window/b2.p2.tsv
new file mode 100644
index 0000000..9e62ba7
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p2.tsv
@@ -0,0 +1,40 @@
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 342
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
+20 391
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p2/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p2/0.data.json b/exec/java-exec/src/test/resources/window/b2.p2/0.data.json
new file mode 100644
index 0000000..3e7efda
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p2/0.data.json
@@ -0,0 +1,22 @@
+// partition rows 20, subs [3, 5, 9]
+// partition rows 20, subs [9, 10]
+{ "employee_id":18, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":31, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":39, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":32, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":8, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":34, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":24, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":1, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":10, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":22, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":36, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":15, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":4, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":2, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":11, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":33, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":37, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":9, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":30, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":23, "position_id":2, "sub":9, "salary":19 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p2/1.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p2/1.data.json b/exec/java-exec/src/test/resources/window/b2.p2/1.data.json
new file mode 100644
index 0000000..acdd419
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p2/1.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":27, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":13, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":21, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":0, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":25, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":3, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":5, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":17, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":12, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":29, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":6, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":7, "position_id":1, "sub":5, "salary":15 }
+{ "employee_id":16, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":35, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":38, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":20, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":19, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":14, "position_id":1, "sub":9, "salary":19 }
+{ "employee_id":28, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":26, "position_id":2, "sub":9, "salary":19 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p4.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p4.subs.tsv b/exec/java-exec/src/test/resources/window/b2.p4.subs.tsv
new file mode 100644
index 0000000..62def92
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p4.subs.tsv
@@ -0,0 +1,40 @@
+1 11 1 1 1 0.2 0.0
+3 35 2 2 2 0.6 0.25
+3 35 3 2 2 0.6 0.25
+5 61 4 4 3 1.0 0.75
+5 61 5 4 3 1.0 0.75
+3 39 1 1 1 0.3 0.0
+3 39 2 1 1 0.3 0.0
+3 39 3 1 1 0.3 0.0
+7 95 4 4 2 0.7 0.3333333333333333
+7 95 5 4 2 0.7 0.3333333333333333
+7 95 6 4 2 0.7 0.3333333333333333
+7 95 7 4 2 0.7 0.3333333333333333
+10 140 8 8 3 1.0 0.7777777777777778
+10 140 9 8 3 1.0 0.7777777777777778
+10 140 10 8 3 1.0 0.7777777777777778
+5 75 1 1 1 0.3333333333333333 0.0
+5 75 2 1 1 0.3333333333333333 0.0
+5 75 3 1 1 0.3333333333333333 0.0
+5 75 4 1 1 0.3333333333333333 0.0
+5 75 5 1 1 0.3333333333333333 0.0
+11 171 6 6 2 0.7333333333333333 0.35714285714285715
+11 171 7 6 2 0.7333333333333333 0.35714285714285715
+11 171 8 6 2 0.7333333333333333 0.35714285714285715
+11 171 9 6 2 0.7333333333333333 0.35714285714285715
+11 171 10 6 2 0.7333333333333333 0.35714285714285715
+11 171 11 6 2 0.7333333333333333 0.35714285714285715
+15 239 12 12 3 1.0 0.7857142857142857
+15 239 13 12 3 1.0 0.7857142857142857
+15 239 14 12 3 1.0 0.7857142857142857
+15 239 15 12 3 1.0 0.7857142857142857
+7 119 1 1 1 0.7 0.0
+7 119 2 1 1 0.7 0.0
+7 119 3 1 1 0.7 0.0
+7 119 4 1 1 0.7 0.0
+7 119 5 1 1 0.7 0.0
+7 119 6 1 1 0.7 0.0
+7 119 7 1 1 0.7 0.0
+10 173 8 8 2 1.0 0.7777777777777778
+10 173 9 8 2 1.0 0.7777777777777778
+10 173 10 8 2 1.0 0.7777777777777778
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p4.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p4.tsv b/exec/java-exec/src/test/resources/window/b2.p4.tsv
new file mode 100644
index 0000000..a9302c6
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p4.tsv
@@ -0,0 +1,40 @@
+5 61
+5 61
+5 61
+5 61
+5 61
+10 140
+10 140
+10 140
+10 140
+10 140
+10 140
+10 140
+10 140
+10 140
+10 140
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+15 239
+10 173
+10 173
+10 173
+10 173
+10 173
+10 173
+10 173
+10 173
+10 173
+10 173
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p4/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p4/0.data.json b/exec/java-exec/src/test/resources/window/b2.p4/0.data.json
new file mode 100644
index 0000000..02da7b8
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p4/0.data.json
@@ -0,0 +1,24 @@
+// partition rows 5, subs [1, 2, 3]
+// partition rows 10, subs [3, 4, 5]
+// partition rows 15, subs [5, 6, 7]
+// partition rows 10, subs [7, 8]
+{ "employee_id":3, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":31, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":2, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":37, "position_id":4, "sub":8, "salary":18 }
+{ "employee_id":13, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":15, "position_id":3, "sub":5, "salary":15 }
+{ "employee_id":26, "position_id":3, "sub":7, "salary":17 }
+{ "employee_id":38, "position_id":4, "sub":8, "salary":18 }
+{ "employee_id":4, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":16, "position_id":3, "sub":5, "salary":15 }
+{ "employee_id":21, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":12, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":29, "position_id":3, "sub":7, "salary":17 }
+{ "employee_id":5, "position_id":2, "sub":3, "salary":13 }
+{ "employee_id":14, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":30, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":27, "position_id":3, "sub":7, "salary":17 }
+{ "employee_id":10, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":33, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":6, "position_id":2, "sub":3, "salary":13 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b2.p4/1.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b2.p4/1.data.json b/exec/java-exec/src/test/resources/window/b2.p4/1.data.json
new file mode 100644
index 0000000..3ed76af
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b2.p4/1.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":1, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":36, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":19, "position_id":3, "sub":5, "salary":15 }
+{ "employee_id":22, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":34, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":11, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":23, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":9, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":28, "position_id":3, "sub":7, "salary":17 }
+{ "employee_id":18, "position_id":3, "sub":5, "salary":15 }
+{ "employee_id":32, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":8, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":17, "position_id":3, "sub":5, "salary":15 }
+{ "employee_id":0, "position_id":1, "sub":1, "salary":11 }
+{ "employee_id":39, "position_id":4, "sub":8, "salary":18 }
+{ "employee_id":20, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":25, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":24, "position_id":3, "sub":6, "salary":16 }
+{ "employee_id":35, "position_id":4, "sub":7, "salary":17 }
+{ "employee_id":7, "position_id":2, "sub":3, "salary":13 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b3.p2.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b3.p2.subs.tsv b/exec/java-exec/src/test/resources/window/b3.p2.subs.tsv
new file mode 100644
index 0000000..3e735aa
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b3.p2.subs.tsv
@@ -0,0 +1,60 @@
+1 11 1 1 1 0.2 0.0
+3 35 2 2 2 0.6 0.25
+3 35 3 2 2 0.6 0.25
+5 61 4 4 3 1.0 0.75
+5 61 5 4 3 1.0 0.75
+4 56 1 1 1 0.07272727272727272 0.0
+4 56 2 1 1 0.07272727272727272 0.0
+4 56 3 1 1 0.07272727272727272 0.0
+4 56 4 1 1 0.07272727272727272 0.0
+9 131 5 5 2 0.16363636363636364 0.07407407407407407
+9 131 6 5 2 0.16363636363636364 0.07407407407407407
+9 131 7 5 2 0.16363636363636364 0.07407407407407407
+9 131 8 5 2 0.16363636363636364 0.07407407407407407
+9 131 9 5 2 0.16363636363636364 0.07407407407407407
+16 250 10 10 3 0.2909090909090909 0.16666666666666666
+16 250 11 10 3 0.2909090909090909 0.16666666666666666
+16 250 12 10 3 0.2909090909090909 0.16666666666666666
+16 250 13 10 3 0.2909090909090909 0.16666666666666666
+16 250 14 10 3 0.2909090909090909 0.16666666666666666
+16 250 15 10 3 0.2909090909090909 0.16666666666666666
+16 250 16 10 3 0.2909090909090909 0.16666666666666666
+24 394 17 17 4 0.43636363636363634 0.2962962962962963
+24 394 18 17 4 0.43636363636363634 0.2962962962962963
+24 394 19 17 4 0.43636363636363634 0.2962962962962963
+24 394 20 17 4 0.43636363636363634 0.2962962962962963
+24 394 21 17 4 0.43636363636363634 0.2962962962962963
+24 394 22 17 4 0.43636363636363634 0.2962962962962963
+24 394 23 17 4 0.43636363636363634 0.2962962962962963
+24 394 24 17 4 0.43636363636363634 0.2962962962962963
+33 565 25 25 5 0.6 0.4444444444444444
+33 565 26 25 5 0.6 0.4444444444444444
+33 565 27 25 5 0.6 0.4444444444444444
+33 565 28 25 5 0.6 0.4444444444444444
+33 565 29 25 5 0.6 0.4444444444444444
+33 565 30 25 5 0.6 0.4444444444444444
+33 565 31 25 5 0.6 0.4444444444444444
+33 565 32 25 5 0.6 0.4444444444444444
+33 565 33 25 5 0.6 0.4444444444444444
+43 765 34 34 6 0.7818181818181819 0.6111111111111112
+43 765 35 34 6 0.7818181818181819 0.6111111111111112
+43 765 36 34 6 0.7818181818181819 0.6111111111111112
+43 765 37 34 6 0.7818181818181819 0.6111111111111112
+43 765 38 34 6 0.7818181818181819 0.6111111111111112
+43 765 39 34 6 0.7818181818181819 0.6111111111111112
+43 765 40 34 6 0.7818181818181819 0.6111111111111112
+43 765 41 34 6 0.7818181818181819 0.6111111111111112
+43 765 42 34 6 0.7818181818181819 0.6111111111111112
+43 765 43 34 6 0.7818181818181819 0.6111111111111112
+54 996 44 44 7 0.9818181818181818 0.7962962962962963
+54 996 45 44 7 0.9818181818181818 0.7962962962962963
+54 996 46 44 7 0.9818181818181818 0.7962962962962963
+54 996 47 44 7 0.9818181818181818 0.7962962962962963
+54 996 48 44 7 0.9818181818181818 0.7962962962962963
+54 996 49 44 7 0.9818181818181818 0.7962962962962963
+54 996 50 44 7 0.9818181818181818 0.7962962962962963
+54 996 51 44 7 0.9818181818181818 0.7962962962962963
+54 996 52 44 7 0.9818181818181818 0.7962962962962963
+54 996 53 44 7 0.9818181818181818 0.7962962962962963
+54 996 54 44 7 0.9818181818181818 0.7962962962962963
+55 1018 55 55 8 1.0 1.0
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b3.p2.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b3.p2.tsv b/exec/java-exec/src/test/resources/window/b3.p2.tsv
new file mode 100644
index 0000000..ef32c2e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b3.p2.tsv
@@ -0,0 +1,60 @@
+5 61
+5 61
+5 61
+5 61
+5 61
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
+55 1018
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b3.p2/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b3.p2/0.data.json b/exec/java-exec/src/test/resources/window/b3.p2/0.data.json
new file mode 100644
index 0000000..39e285f
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b3.p2/0.data.json
@@ -0,0 +1,22 @@
+// partition rows 5, subs [1, 2, 3]
+// partition rows 55, subs [4, 5, 7, 8, 9, 10, 11, 12]
+{ "employee_id":56, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":42, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":53, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":28, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":33, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":25, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":23, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":41, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":50, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":27, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":35, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":54, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":4, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":9, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":15, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":13, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":31, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":12, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":57, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":45, "position_id":2, "sub":10, "salary":20 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b3.p2/1.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b3.p2/1.data.json b/exec/java-exec/src/test/resources/window/b3.p2/1.data.json
new file mode 100644
index 0000000..1d92e1a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b3.p2/1.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":26, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":6, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":19, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":7, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":51, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":47, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":21, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":44, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":46, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":59, "position_id":2, "sub":12, "salary":22 }
+{ "employee_id":36, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":30, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":1, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":5, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":40, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":32, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":11, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":0, "position_id":1, "sub":1, "salary":11 }
+{ "employee_id":49, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":48, "position_id":2, "sub":11, "salary":21 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b3.p2/2.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b3.p2/2.data.json b/exec/java-exec/src/test/resources/window/b3.p2/2.data.json
new file mode 100644
index 0000000..e0f5f3c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b3.p2/2.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":43, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":18, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":52, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":29, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":8, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":58, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":55, "position_id":2, "sub":11, "salary":21 }
+{ "employee_id":16, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":14, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":20, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":37, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":38, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":17, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":2, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":22, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":24, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":39, "position_id":2, "sub":10, "salary":20 }
+{ "employee_id":34, "position_id":2, "sub":9, "salary":19 }
+{ "employee_id":3, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":10, "position_id":2, "sub":5, "salary":15 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4.subs.tsv b/exec/java-exec/src/test/resources/window/b4.p4.subs.tsv
new file mode 100644
index 0000000..c13597b
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4.subs.tsv
@@ -0,0 +1,80 @@
+1 11 1 1 1 0.1 0.0
+3 35 2 2 2 0.3 0.1111111111111111
+3 35 3 2 2 0.3 0.1111111111111111
+10 126 4 4 3 1.0 0.3333333333333333
+10 126 5 4 3 1.0 0.3333333333333333
+10 126 6 4 3 1.0 0.3333333333333333
+10 126 7 4 3 1.0 0.3333333333333333
+10 126 8 4 3 1.0 0.3333333333333333
+10 126 9 4 3 1.0 0.3333333333333333
+10 126 10 4 3 1.0 0.3333333333333333
+3 39 1 1 1 0.1 0.0
+3 39 2 1 1 0.1 0.0
+3 39 3 1 1 0.1 0.0
+7 95 4 4 2 0.23333333333333334 0.10344827586206896
+7 95 5 4 2 0.23333333333333334 0.10344827586206896
+7 95 6 4 2 0.23333333333333334 0.10344827586206896
+7 95 7 4 2 0.23333333333333334 0.10344827586206896
+12 170 8 8 3 0.4 0.2413793103448276
+12 170 9 8 3 0.4 0.2413793103448276
+12 170 10 8 3 0.4 0.2413793103448276
+12 170 11 8 3 0.4 0.2413793103448276
+12 170 12 8 3 0.4 0.2413793103448276
+18 266 13 13 4 0.6 0.41379310344827586
+18 266 14 13 4 0.6 0.41379310344827586
+18 266 15 13 4 0.6 0.41379310344827586
+18 266 16 13 4 0.6 0.41379310344827586
+18 266 17 13 4 0.6 0.41379310344827586
+18 266 18 13 4 0.6 0.41379310344827586
+25 385 19 19 5 0.8333333333333334 0.6206896551724138
+25 385 20 19 5 0.8333333333333334 0.6206896551724138
+25 385 21 19 5 0.8333333333333334 0.6206896551724138
+25 385 22 19 5 0.8333333333333334 0.6206896551724138
+25 385 23 19 5 0.8333333333333334 0.6206896551724138
+25 385 24 19 5 0.8333333333333334 0.6206896551724138
+25 385 25 19 5 0.8333333333333334 0.6206896551724138
+30 475 26 26 6 1.0 0.8620689655172413
+30 475 27 26 6 1.0 0.8620689655172413
+30 475 28 26 6 1.0 0.8620689655172413
+30 475 29 26 6 1.0 0.8620689655172413
+30 475 30 26 6 1.0 0.8620689655172413
+8 144 1 1 1 0.4 0.0
+8 144 2 1 1 0.4 0.0
+8 144 3 1 1 0.4 0.0
+8 144 4 1 1 0.4 0.0
+8 144 5 1 1 0.4 0.0
+8 144 6 1 1 0.4 0.0
+8 144 7 1 1 0.4 0.0
+8 144 8 1 1 0.4 0.0
+17 315 9 9 2 0.85 0.42105263157894735
+17 315 10 9 2 0.85 0.42105263157894735
+17 315 11 9 2 0.85 0.42105263157894735
+17 315 12 9 2 0.85 0.42105263157894735
+17 315 13 9 2 0.85 0.42105263157894735
+17 315 14 9 2 0.85 0.42105263157894735
+17 315 15 9 2 0.85 0.42105263157894735
+17 315 16 9 2 0.85 0.42105263157894735
+17 315 17 9 2 0.85 0.42105263157894735
+20 375 18 18 3 1.0 0.8947368421052632
+20 375 19 18 3 1.0 0.8947368421052632
+20 375 20 18 3 1.0 0.8947368421052632
+10 200 1 1 1 0.5 0.0
+10 200 2 1 1 0.5 0.0
+10 200 3 1 1 0.5 0.0
+10 200 4 1 1 0.5 0.0
+10 200 5 1 1 0.5 0.0
+10 200 6 1 1 0.5 0.0
+10 200 7 1 1 0.5 0.0
+10 200 8 1 1 0.5 0.0
+10 200 9 1 1 0.5 0.0
+10 200 10 1 1 0.5 0.0
+20 410 11 11 2 1.0 0.5263157894736842
+20 410 12 11 2 1.0 0.5263157894736842
+20 410 13 11 2 1.0 0.5263157894736842
+20 410 14 11 2 1.0 0.5263157894736842
+20 410 15 11 2 1.0 0.5263157894736842
+20 410 16 11 2 1.0 0.5263157894736842
+20 410 17 11 2 1.0 0.5263157894736842
+20 410 18 11 2 1.0 0.5263157894736842
+20 410 19 11 2 1.0 0.5263157894736842
+20 410 20 11 2 1.0 0.5263157894736842
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4.tsv b/exec/java-exec/src/test/resources/window/b4.p4.tsv
new file mode 100644
index 0000000..29c57ae
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4.tsv
@@ -0,0 +1,80 @@
+10 126
+10 126
+10 126
+10 126
+10 126
+10 126
+10 126
+10 126
+10 126
+10 126
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+30 475
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 375
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
+20 410
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4/0.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4/0.data.json b/exec/java-exec/src/test/resources/window/b4.p4/0.data.json
new file mode 100644
index 0000000..c40697c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4/0.data.json
@@ -0,0 +1,24 @@
+// partition rows 10, subs [1, 2, 3]
+// partition rows 30, subs [3, 4, 5, 6, 7, 8]
+// partition rows 20, subs [8, 9, 10]
+// partition rows 20, subs [10, 11]
+{ "employee_id":37, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":52, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":16, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":20, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":5, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":45, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":40, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":1, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":48, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":13, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":70, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":47, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":24, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":8, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":63, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":32, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":58, "position_id":3, "sub":10, "salary":20 }
+{ "employee_id":10, "position_id":2, "sub":3, "salary":13 }
+{ "employee_id":22, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":61, "position_id":4, "sub":10, "salary":20 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4/1.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4/1.data.json b/exec/java-exec/src/test/resources/window/b4.p4/1.data.json
new file mode 100644
index 0000000..472c9b5
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4/1.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":74, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":6, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":15, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":36, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":34, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":41, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":14, "position_id":2, "sub":4, "salary":14 }
+{ "employee_id":75, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":53, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":0, "position_id":1, "sub":1, "salary":11 }
+{ "employee_id":7, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":23, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":60, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":55, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":46, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":65, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":3, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":59, "position_id":3, "sub":10, "salary":20 }
+{ "employee_id":35, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":69, "position_id":4, "sub":10, "salary":20 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4/2.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4/2.data.json b/exec/java-exec/src/test/resources/window/b4.p4/2.data.json
new file mode 100644
index 0000000..292200c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4/2.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":9, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":38, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":11, "position_id":2, "sub":3, "salary":13 }
+{ "employee_id":42, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":78, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":77, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":21, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":49, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":57, "position_id":3, "sub":10, "salary":20 }
+{ "employee_id":66, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":4, "position_id":1, "sub":3, "salary":13 }
+{ "employee_id":73, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":28, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":71, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":31, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":2, "position_id":1, "sub":2, "salary":12 }
+{ "employee_id":26, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":68, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":64, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":25, "position_id":2, "sub":6, "salary":16 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b4.p4/3.data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b4.p4/3.data.json b/exec/java-exec/src/test/resources/window/b4.p4/3.data.json
new file mode 100644
index 0000000..cd9008c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b4.p4/3.data.json
@@ -0,0 +1,20 @@
+{ "employee_id":33, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":44, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":50, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":67, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":79, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":17, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":54, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":51, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":62, "position_id":4, "sub":10, "salary":20 }
+{ "employee_id":56, "position_id":3, "sub":9, "salary":19 }
+{ "employee_id":43, "position_id":3, "sub":8, "salary":18 }
+{ "employee_id":12, "position_id":2, "sub":3, "salary":13 }
+{ "employee_id":19, "position_id":2, "sub":5, "salary":15 }
+{ "employee_id":29, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":39, "position_id":2, "sub":8, "salary":18 }
+{ "employee_id":30, "position_id":2, "sub":7, "salary":17 }
+{ "employee_id":27, "position_id":2, "sub":6, "salary":16 }
+{ "employee_id":72, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":76, "position_id":4, "sub":11, "salary":21 }
+{ "employee_id":18, "position_id":2, "sub":5, "salary":15 }
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/mediumData.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/mediumData.json b/exec/java-exec/src/test/resources/window/mediumData.json
deleted file mode 100644
index ad86627..0000000
--- a/exec/java-exec/src/test/resources/window/mediumData.json
+++ /dev/null
@@ -1,1000 +0,0 @@
-{"id":814, "a": 1626, "group": 203}
-{"id":425, "a": 849, "group": 106}
-{"id":900, "a": 1800, "group": 225}
-{"id":156, "a": 312, "group": 39}
-{"id":348, "a": 696, "group": 87}
-{"id":987, "a": 1971, "group": 246}
-{"id":255, "a": 507, "group": 63}
-{"id":4, "a": 8, "group": 1}
-{"id":512, "a": 1024, "group": 128}
-{"id":341, "a": 681, "group": 85}
-{"id":113, "a": 225, "group": 28}
-{"id":311, "a": 619, "group": 77}
-{"id":906, "a": 1810, "group": 226}
-{"id":889, "a": 1777, "group": 222}
-{"id":611, "a": 1219, "group": 152}
-{"id":963, "a": 1923, "group": 240}
-{"id":522, "a": 1042, "group": 130}
-{"id":615, "a": 1227, "group": 153}
-{"id":227, "a": 451, "group": 56}
-{"id":365, "a": 729, "group": 91}
-{"id":73, "a": 145, "group": 18}
-{"id":747, "a": 1491, "group": 186}
-{"id":580, "a": 1160, "group": 145}
-{"id":552, "a": 1104, "group": 138}
-{"id":716, "a": 1432, "group": 179}
-{"id":982, "a": 1962, "group": 245}
-{"id":118, "a": 234, "group": 29}
-{"id":639, "a": 1275, "group": 159}
-{"id":273, "a": 545, "group": 68}
-{"id":679, "a": 1355, "group": 169}
-{"id":338, "a": 674, "group": 84}
-{"id":402, "a": 802, "group": 100}
-{"id":476, "a": 952, "group": 119}
-{"id":628, "a": 1256, "group": 157}
-{"id":325, "a": 649, "group": 81}
-{"id":749, "a": 1497, "group": 187}
-{"id":912, "a": 1824, "group": 228}
-{"id":995, "a": 1987, "group": 248}
-{"id":605, "a": 1209, "group": 151}
-{"id":141, "a": 281, "group": 35}
-{"id":700, "a": 1400, "group": 175}
-{"id":61, "a": 121, "group": 15}
-{"id":478, "a": 954, "group": 119}
-{"id":556, "a": 1112, "group": 139}
-{"id":229, "a": 457, "group": 57}
-{"id":487, "a": 971, "group": 121}
-{"id":824, "a": 1648, "group": 206}
-{"id":431, "a": 859, "group": 107}
-{"id":443, "a": 883, "group": 110}
-{"id":135, "a": 267, "group": 33}
-{"id":417, "a": 833, "group": 104}
-{"id":980, "a": 1960, "group": 245}
-{"id":785, "a": 1569, "group": 196}
-{"id":917, "a": 1833, "group": 229}
-{"id":656, "a": 1312, "group": 164}
-{"id":210, "a": 418, "group": 52}
-{"id":196, "a": 392, "group": 49}
-{"id":361, "a": 721, "group": 90}
-{"id":281, "a": 561, "group": 70}
-{"id":550, "a": 1098, "group": 137}
-{"id":558, "a": 1114, "group": 139}
-{"id":677, "a": 1353, "group": 169}
-{"id":604, "a": 1208, "group": 151}
-{"id":8, "a": 16, "group": 2}
-{"id":290, "a": 578, "group": 72}
-{"id":932, "a": 1864, "group": 233}
-{"id":731, "a": 1459, "group": 182}
-{"id":477, "a": 953, "group": 119}
-{"id":859, "a": 1715, "group": 214}
-{"id":291, "a": 579, "group": 72}
-{"id":531, "a": 1059, "group": 132}
-{"id":499, "a": 995, "group": 124}
-{"id":389, "a": 777, "group": 97}
-{"id":182, "a": 362, "group": 45}
-{"id":959, "a": 1915, "group": 239}
-{"id":523, "a": 1043, "group": 130}
-{"id":81, "a": 161, "group": 20}
-{"id":439, "a": 875, "group": 109}
-{"id":228, "a": 456, "group": 57}
-{"id":301, "a": 601, "group": 75}
-{"id":208, "a": 416, "group": 52}
-{"id":370, "a": 738, "group": 92}
-{"id":383, "a": 763, "group": 95}
-{"id":209, "a": 417, "group": 52}
-{"id":462, "a": 922, "group": 115}
-{"id":729, "a": 1457, "group": 182}
-{"id":602, "a": 1202, "group": 150}
-{"id":936, "a": 1872, "group": 234}
-{"id":750, "a": 1498, "group": 187}
-{"id":871, "a": 1739, "group": 217}
-{"id":120, "a": 240, "group": 30}
-{"id":843, "a": 1683, "group": 210}
-{"id":260, "a": 520, "group": 65}
-{"id":240, "a": 480, "group": 60}
-{"id":976, "a": 1952, "group": 244}
-{"id":344, "a": 688, "group": 86}
-{"id":385, "a": 769, "group": 96}
-{"id":410, "a": 818, "group": 102}
-{"id":931, "a": 1859, "group": 232}
-{"id":891, "a": 1779, "group": 222}
-{"id":745, "a": 1489, "group": 186}
-{"id":813, "a": 1625, "group": 203}
-{"id":129, "a": 257, "group": 32}
-{"id":596, "a": 1192, "group": 149}
-{"id":517, "a": 1033, "group": 129}
-{"id":755, "a": 1507, "group": 188}
-{"id":663, "a": 1323, "group": 165}
-{"id":233, "a": 465, "group": 58}
-{"id":401, "a": 801, "group": 100}
-{"id":473, "a": 945, "group": 118}
-{"id":990, "a": 1978, "group": 247}
-{"id":384, "a": 768, "group": 96}
-{"id":178, "a": 354, "group": 44}
-{"id":446, "a": 890, "group": 111}
-{"id":828, "a": 1656, "group": 207}
-{"id":356, "a": 712, "group": 89}
-{"id":249, "a": 497, "group": 62}
-{"id":553, "a": 1105, "group": 138}
-{"id":378, "a": 754, "group": 94}
-{"id":126, "a": 250, "group": 31}
-{"id":806, "a": 1610, "group": 201}
-{"id":540, "a": 1080, "group": 135}
-{"id":545, "a": 1089, "group": 136}
-{"id":398, "a": 794, "group": 99}
-{"id":848, "a": 1696, "group": 212}
-{"id":493, "a": 985, "group": 123}
-{"id":928, "a": 1856, "group": 232}
-{"id":408, "a": 816, "group": 102}
-{"id":285, "a": 569, "group": 71}
-{"id":795, "a": 1587, "group": 198}
-{"id":74, "a": 146, "group": 18}
-{"id":332, "a": 664, "group": 83}
-{"id":712, "a": 1424, "group": 178}
-{"id":858, "a": 1714, "group": 214}
-{"id":961, "a": 1921, "group": 240}
-{"id":212, "a": 424, "group": 53}
-{"id":11, "a": 19, "group": 2}
-{"id":839, "a": 1675, "group": 209}
-{"id":302, "a": 602, "group": 75}
-{"id":117, "a": 233, "group": 29}
-{"id":852, "a": 1704, "group": 213}
-{"id":528, "a": 1056, "group": 132}
-{"id":829, "a": 1657, "group": 207}
-{"id":563, "a": 1123, "group": 140}
-{"id":968, "a": 1936, "group": 242}
-{"id":658, "a": 1314, "group": 164}
-{"id":49, "a": 97, "group": 12}
-{"id":52, "a": 104, "group": 13}
-{"id":186, "a": 370, "group": 46}
-{"id":407, "a": 811, "group": 101}
-{"id":98, "a": 194, "group": 24}
-{"id":377, "a": 753, "group": 94}
-{"id":195, "a": 387, "group": 48}
-{"id":826, "a": 1650, "group": 206}
-{"id":783, "a": 1563, "group": 195}
-{"id":284, "a": 568, "group": 71}
-{"id":34, "a": 66, "group": 8}
-{"id":752, "a": 1504, "group": 188}
-{"id":472, "a": 944, "group": 118}
-{"id":500, "a": 1000, "group": 125}
-{"id":812, "a": 1624, "group": 203}
-{"id":300, "a": 600, "group": 75}
-{"id":691, "a": 1379, "group": 172}
-{"id":435, "a": 867, "group": 108}
-{"id":693, "a": 1385, "group": 173}
-{"id":847, "a": 1691, "group": 211}
-{"id":235, "a": 467, "group": 58}
-{"id":45, "a": 89, "group": 11}
-{"id":947, "a": 1891, "group": 236}
-{"id":184, "a": 368, "group": 46}
-{"id":996, "a": 1992, "group": 249}
-{"id":150, "a": 298, "group": 37}
-{"id":413, "a": 825, "group": 103}
-{"id":952, "a": 1904, "group": 238}
-{"id":594, "a": 1186, "group": 148}
-{"id":133, "a": 265, "group": 33}
-{"id":587, "a": 1171, "group": 146}
-{"id":612, "a": 1224, "group": 153}
-{"id":515, "a": 1027, "group": 128}
-{"id":718, "a": 1434, "group": 179}
-{"id":884, "a": 1768, "group": 221}
-{"id":887, "a": 1771, "group": 221}
-{"id":585, "a": 1169, "group": 146}
-{"id":695, "a": 1387, "group": 173}
-{"id":965, "a": 1929, "group": 241}
-{"id":591, "a": 1179, "group": 147}
-{"id":374, "a": 746, "group": 93}
-{"id":780, "a": 1560, "group": 195}
-{"id":305, "a": 609, "group": 76}
-{"id":71, "a": 139, "group": 17}
-{"id":84, "a": 168, "group": 21}
-{"id":58, "a": 114, "group": 14}
-{"id":12, "a": 24, "group": 3}
-{"id":315, "a": 627, "group": 78}
-{"id":131, "a": 259, "group": 32}
-{"id":362, "a": 722, "group": 90}
-{"id":490, "a": 978, "group": 122}
-{"id":234, "a": 466, "group": 58}
-{"id":349, "a": 697, "group": 87}
-{"id":688, "a": 1376, "group": 172}
-{"id":379, "a": 755, "group": 94}
-{"id":561, "a": 1121, "group": 140}
-{"id":363, "a": 723, "group": 90}
-{"id":287, "a": 571, "group": 71}
-{"id":770, "a": 1538, "group": 192}
-{"id":127, "a": 251, "group": 31}
-{"id":583, "a": 1163, "group": 145}
-{"id":471, "a": 939, "group": 117}
-{"id":788, "a": 1576, "group": 197}
-{"id":897, "a": 1793, "group": 224}
-{"id":916, "a": 1832, "group": 229}
-{"id":956, "a": 1912, "group": 239}
-{"id":224, "a": 448, "group": 56}
-{"id":787, "a": 1571, "group": 196}
-{"id":173, "a": 345, "group": 43}
-{"id":47, "a": 91, "group": 11}
-{"id":180, "a": 360, "group": 45}
-{"id":488, "a": 976, "group": 122}
-{"id":764, "a": 1528, "group": 191}
-{"id":112, "a": 224, "group": 28}
-{"id":781, "a": 1561, "group": 195}
-{"id":14, "a": 26, "group": 3}
-{"id":204, "a": 408, "group": 51}
-{"id":317, "a": 633, "group": 79}
-{"id":784, "a": 1568, "group": 196}
-{"id":796, "a": 1592, "group": 199}
-{"id":375, "a": 747, "group": 93}
-{"id":618, "a": 1234, "group": 154}
-{"id":207, "a": 411, "group": 51}
-{"id":179, "a": 355, "group": 44}
-{"id":297, "a": 593, "group": 74}
-{"id":838, "a": 1674, "group": 209}
-{"id":699, "a": 1395, "group": 174}
-{"id":320, "a": 640, "group": 80}
-{"id":675, "a": 1347, "group": 168}
-{"id":925, "a": 1849, "group": 231}
-{"id":684, "a": 1368, "group": 171}
-{"id":986, "a": 1970, "group": 246}
-{"id":930, "a": 1858, "group": 232}
-{"id":911, "a": 1819, "group": 227}
-{"id":977, "a": 1953, "group": 244}
-{"id":48, "a": 96, "group": 12}
-{"id":496, "a": 992, "group": 124}
-{"id":794, "a": 1586, "group": 198}
-{"id":867, "a": 1731, "group": 216}
-{"id":520, "a": 1040, "group": 130}
-{"id":621, "a": 1241, "group": 155}
-{"id":475, "a": 947, "group": 118}
-{"id":270, "a": 538, "group": 67}
-{"id":648, "a": 1296, "group": 162}
-{"id":842, "a": 1682, "group": 210}
-{"id":200, "a": 400, "group": 50}
-{"id":924, "a": 1848, "group": 231}
-{"id":466, "a": 930, "group": 116}
-{"id":40, "a": 80, "group": 10}
-{"id":600, "a": 1200, "group": 150}
-{"id":883, "a": 1763, "group": 220}
-{"id":221, "a": 441, "group": 55}
-{"id":106, "a": 210, "group": 26}
-{"id":313, "a": 625, "group": 78}
-{"id":761, "a": 1521, "group": 190}
-{"id":800, "a": 1600, "group": 200}
-{"id":241, "a": 481, "group": 60}
-{"id":640, "a": 1280, "group": 160}
-{"id":358, "a": 714, "group": 89}
-{"id":960, "a": 1920, "group": 240}
-{"id":347, "a": 691, "group": 86}
-{"id":646, "a": 1290, "group": 161}
-{"id":236, "a": 472, "group": 59}
-{"id":920, "a": 1840, "group": 230}
-{"id":586, "a": 1170, "group": 146}
-{"id":175, "a": 347, "group": 43}
-{"id":371, "a": 739, "group": 92}
-{"id":741, "a": 1481, "group": 185}
-{"id":652, "a": 1304, "group": 163}
-{"id":164, "a": 328, "group": 41}
-{"id":444, "a": 888, "group": 111}
-{"id":949, "a": 1897, "group": 237}
-{"id":115, "a": 227, "group": 28}
-{"id":893, "a": 1785, "group": 223}
-{"id":940, "a": 1880, "group": 235}
-{"id":261, "a": 521, "group": 65}
-{"id":105, "a": 209, "group": 26}
-{"id":449, "a": 897, "group": 112}
-{"id":94, "a": 186, "group": 23}
-{"id":810, "a": 1618, "group": 202}
-{"id":252, "a": 504, "group": 63}
-{"id":946, "a": 1890, "group": 236}
-{"id":136, "a": 272, "group": 34}
-{"id":70, "a": 138, "group": 17}
-{"id":203, "a": 403, "group": 50}
-{"id":276, "a": 552, "group": 69}
-{"id":703, "a": 1403, "group": 175}
-{"id":714, "a": 1426, "group": 178}
-{"id":144, "a": 288, "group": 36}
-{"id":763, "a": 1523, "group": 190}
-{"id":142, "a": 282, "group": 35}
-{"id":406, "a": 810, "group": 101}
-{"id":225, "a": 449, "group": 56}
-{"id":93, "a": 185, "group": 23}
-{"id":622, "a": 1242, "group": 155}
-{"id":461, "a": 921, "group": 115}
-{"id":923, "a": 1843, "group": 230}
-{"id":971, "a": 1939, "group": 242}
-{"id":748, "a": 1496, "group": 187}
-{"id":687, "a": 1371, "group": 171}
-{"id":340, "a": 680, "group": 85}
-{"id":223, "a": 443, "group": 55}
-{"id":625, "a": 1249, "group": 156}
-{"id":895, "a": 1787, "group": 223}
-{"id":738, "a": 1474, "group": 184}
-{"id":35, "a": 67, "group": 8}
-{"id":159, "a": 315, "group": 39}
-{"id":981, "a": 1961, "group": 245}
-{"id":521, "a": 1041, "group": 130}
-{"id":36, "a": 72, "group": 9}
-{"id":360, "a": 720, "group": 90}
-{"id":194, "a": 386, "group": 48}
-{"id":333, "a": 665, "group": 83}
-{"id":816, "a": 1632, "group": 204}
-{"id":805, "a": 1609, "group": 201}
-{"id":122, "a": 242, "group": 30}
-{"id":67, "a": 131, "group": 16}
-{"id":866, "a": 1730, "group": 216}
-{"id":219, "a": 435, "group": 54}
-{"id":274, "a": 546, "group": 68}
-{"id":102, "a": 202, "group": 25}
-{"id":951, "a": 1899, "group": 237}
-{"id":836, "a": 1672, "group": 209}
-{"id":191, "a": 379, "group": 47}
-{"id":337, "a": 673, "group": 84}
-{"id":841, "a": 1681, "group": 210}
-{"id":92, "a": 184, "group": 23}
-{"id":481, "a": 961, "group": 120}
-{"id":970, "a": 1938, "group": 242}
-{"id":878, "a": 1754, "group": 219}
-{"id":294, "a": 586, "group": 73}
-{"id":386, "a": 770, "group": 96}
-{"id":484, "a": 968, "group": 121}
-{"id":789, "a": 1577, "group": 197}
-{"id":492, "a": 984, "group": 123}
-{"id":19, "a": 35, "group": 4}
-{"id":263, "a": 523, "group": 65}
-{"id":514, "a": 1026, "group": 128}
-{"id":352, "a": 704, "group": 88}
-{"id":503, "a": 1003, "group": 125}
-{"id":726, "a": 1450, "group": 181}
-{"id":890, "a": 1778, "group": 222}
-{"id":926, "a": 1850, "group": 231}
-{"id":707, "a": 1411, "group": 176}
-{"id":216, "a": 432, "group": 54}
-{"id":807, "a": 1611, "group": 201}
-{"id":942, "a": 1882, "group": 235}
-{"id":678, "a": 1354, "group": 169}
-{"id":354, "a": 706, "group": 88}
-{"id":77, "a": 153, "group": 19}
-{"id":75, "a": 147, "group": 18}
-{"id":830, "a": 1658, "group": 207}
-{"id":215, "a": 427, "group": 53}
-{"id":966, "a": 1930, "group": 241}
-{"id":603, "a": 1203, "group": 150}
-{"id":137, "a": 273, "group": 34}
-{"id":17, "a": 33, "group": 4}
-{"id":991, "a": 1979, "group": 247}
-{"id":299, "a": 595, "group": 74}
-{"id":643, "a": 1283, "group": 160}
-{"id":190, "a": 378, "group": 47}
-{"id":967, "a": 1931, "group": 241}
-{"id":169, "a": 337, "group": 42}
-{"id":460, "a": 920, "group": 115}
-{"id":330, "a": 658, "group": 82}
-{"id":436, "a": 872, "group": 109}
-{"id":393, "a": 785, "group": 98}
-{"id":329, "a": 657, "group": 82}
-{"id":80, "a": 160, "group": 20}
-{"id":395, "a": 787, "group": 98}
-{"id":623, "a": 1243, "group": 155}
-{"id":110, "a": 218, "group": 27}
-{"id":213, "a": 425, "group": 53}
-{"id":448, "a": 896, "group": 112}
-{"id":671, "a": 1339, "group": 167}
-{"id":751, "a": 1499, "group": 187}
-{"id":606, "a": 1210, "group": 151}
-{"id":624, "a": 1248, "group": 156}
-{"id":766, "a": 1530, "group": 191}
-{"id":31, "a": 59, "group": 7}
-{"id":649, "a": 1297, "group": 162}
-{"id":863, "a": 1723, "group": 215}
-{"id":328, "a": 656, "group": 82}
-{"id":686, "a": 1370, "group": 171}
-{"id":343, "a": 683, "group": 85}
-{"id":418, "a": 834, "group": 104}
-{"id":850, "a": 1698, "group": 212}
-{"id":892, "a": 1784, "group": 223}
-{"id":657, "a": 1313, "group": 164}
-{"id":880, "a": 1760, "group": 220}
-{"id":988, "a": 1976, "group": 247}
-{"id":772, "a": 1544, "group": 193}
-{"id":909, "a": 1817, "group": 227}
-{"id":394, "a": 786, "group": 98}
-{"id":999, "a": 1995, "group": 249}
-{"id":161, "a": 321, "group": 40}
-{"id":754, "a": 1506, "group": 188}
-{"id":56, "a": 112, "group": 14}
-{"id":733, "a": 1465, "group": 183}
-{"id":870, "a": 1738, "group": 217}
-{"id":456, "a": 912, "group": 114}
-{"id":114, "a": 226, "group": 28}
-{"id":571, "a": 1139, "group": 142}
-{"id":567, "a": 1131, "group": 141}
-{"id":827, "a": 1651, "group": 206}
-{"id":757, "a": 1513, "group": 189}
-{"id":720, "a": 1440, "group": 180}
-{"id":709, "a": 1417, "group": 177}
-{"id":831, "a": 1659, "group": 207}
-{"id":773, "a": 1545, "group": 193}
-{"id":201, "a": 401, "group": 50}
-{"id":23, "a": 43, "group": 5}
-{"id":421, "a": 841, "group": 105}
-{"id":516, "a": 1032, "group": 129}
-{"id":22, "a": 42, "group": 5}
-{"id":538, "a": 1074, "group": 134}
-{"id":588, "a": 1176, "group": 147}
-{"id":326, "a": 650, "group": 81}
-{"id":815, "a": 1627, "group": 203}
-{"id":319, "a": 635, "group": 79}
-{"id":440, "a": 880, "group": 110}
-{"id":875, "a": 1747, "group": 218}
-{"id":634, "a": 1266, "group": 158}
-{"id":172, "a": 344, "group": 43}
-{"id":694, "a": 1386, "group": 173}
-{"id":767, "a": 1531, "group": 191}
-{"id":324, "a": 648, "group": 81}
-{"id":33, "a": 65, "group": 8}
-{"id":935, "a": 1867, "group": 233}
-{"id":667, "a": 1331, "group": 166}
-{"id":91, "a": 179, "group": 22}
-{"id":719, "a": 1435, "group": 179}
-{"id":582, "a": 1162, "group": 145}
-{"id":739, "a": 1475, "group": 184}
-{"id":635, "a": 1267, "group": 158}
-{"id":367, "a": 731, "group": 91}
-{"id":636, "a": 1272, "group": 159}
-{"id":743, "a": 1483, "group": 185}
-{"id":463, "a": 923, "group": 115}
-{"id":834, "a": 1666, "group": 208}
-{"id":532, "a": 1064, "group": 133}
-{"id":704, "a": 1408, "group": 176}
-{"id":387, "a": 771, "group": 96}
-{"id":57, "a": 113, "group": 14}
-{"id":153, "a": 305, "group": 38}
-{"id":364, "a": 728, "group": 91}
-{"id":905, "a": 1809, "group": 226}
-{"id":578, "a": 1154, "group": 144}
-{"id":265, "a": 529, "group": 66}
-{"id":642, "a": 1282, "group": 160}
-{"id":689, "a": 1377, "group": 172}
-{"id":574, "a": 1146, "group": 143}
-{"id":318, "a": 634, "group": 79}
-{"id":519, "a": 1035, "group": 129}
-{"id":411, "a": 819, "group": 102}
-{"id":465, "a": 929, "group": 116}
-{"id":174, "a": 346, "group": 43}
-{"id":286, "a": 570, "group": 71}
-{"id":162, "a": 322, "group": 40}
-{"id":894, "a": 1786, "group": 223}
-{"id":445, "a": 889, "group": 111}
-{"id":295, "a": 587, "group": 73}
-{"id":599, "a": 1195, "group": 149}
-{"id":1000, "a": 2000, "group": 250}
-{"id":491, "a": 979, "group": 122}
-{"id":539, "a": 1075, "group": 134}
-{"id":664, "a": 1328, "group": 166}
-{"id":771, "a": 1539, "group": 192}
-{"id":244, "a": 488, "group": 61}
-{"id":123, "a": 243, "group": 30}
-{"id":230, "a": 458, "group": 57}
-{"id":149, "a": 297, "group": 37}
-{"id":467, "a": 931, "group": 116}
-{"id":372, "a": 744, "group": 93}
-{"id":921, "a": 1841, "group": 230}
-{"id":388, "a": 776, "group": 97}
-{"id":898, "a": 1794, "group": 224}
-{"id":239, "a": 475, "group": 59}
-{"id":390, "a": 778, "group": 97}
-{"id":903, "a": 1803, "group": 225}
-{"id":382, "a": 762, "group": 95}
-{"id":715, "a": 1427, "group": 178}
-{"id":774, "a": 1546, "group": 193}
-{"id":259, "a": 515, "group": 64}
-{"id":419, "a": 835, "group": 104}
-{"id":51, "a": 99, "group": 12}
-{"id":929, "a": 1857, "group": 232}
-{"id":455, "a": 907, "group": 113}
-{"id":404, "a": 808, "group": 101}
-{"id":526, "a": 1050, "group": 131}
-{"id":985, "a": 1969, "group": 246}
-{"id":518, "a": 1034, "group": 129}
-{"id":323, "a": 643, "group": 80}
-{"id":821, "a": 1641, "group": 205}
-{"id":427, "a": 851, "group": 106}
-{"id":833, "a": 1665, "group": 208}
-{"id":723, "a": 1443, "group": 180}
-{"id":973, "a": 1945, "group": 243}
-{"id":555, "a": 1107, "group": 138}
-{"id":513, "a": 1025, "group": 128}
-{"id":251, "a": 499, "group": 62}
-{"id":217, "a": 433, "group": 54}
-{"id":581, "a": 1161, "group": 145}
-{"id":345, "a": 689, "group": 86}
-{"id":498, "a": 994, "group": 124}
-{"id":637, "a": 1273, "group": 159}
-{"id":955, "a": 1907, "group": 238}
-{"id":680, "a": 1360, "group": 170}
-{"id":310, "a": 618, "group": 77}
-{"id":817, "a": 1633, "group": 204}
-{"id":346, "a": 690, "group": 86}
-{"id":958, "a": 1914, "group": 239}
-{"id":506, "a": 1010, "group": 126}
-{"id":403, "a": 803, "group": 100}
-{"id":865, "a": 1729, "group": 216}
-{"id":666, "a": 1330, "group": 166}
-{"id":264, "a": 528, "group": 66}
-{"id":258, "a": 514, "group": 64}
-{"id":944, "a": 1888, "group": 236}
-{"id":422, "a": 842, "group": 105}
-{"id":914, "a": 1826, "group": 228}
-{"id":862, "a": 1722, "group": 215}
-{"id":405, "a": 809, "group": 101}
-{"id":250, "a": 498, "group": 62}
-{"id":541, "a": 1081, "group": 135}
-{"id":644, "a": 1288, "group": 161}
-{"id":423, "a": 843, "group": 105}
-{"id":480, "a": 960, "group": 120}
-{"id":28, "a": 56, "group": 7}
-{"id":734, "a": 1466, "group": 183}
-{"id":452, "a": 904, "group": 113}
-{"id":268, "a": 536, "group": 67}
-{"id":708, "a": 1416, "group": 177}
-{"id":451, "a": 899, "group": 112}
-{"id":535, "a": 1067, "group": 133}
-{"id":1, "a": 1, "group": 0}
-{"id":943, "a": 1883, "group": 235}
-{"id":510, "a": 1018, "group": 127}
-{"id":464, "a": 928, "group": 116}
-{"id":705, "a": 1409, "group": 176}
-{"id":381, "a": 761, "group": 95}
-{"id":6, "a": 10, "group": 1}
-{"id":257, "a": 513, "group": 64}
-{"id":851, "a": 1699, "group": 212}
-{"id":938, "a": 1874, "group": 234}
-{"id":835, "a": 1667, "group": 208}
-{"id":501, "a": 1001, "group": 125}
-{"id":296, "a": 592, "group": 74}
-{"id":818, "a": 1634, "group": 204}
-{"id":577, "a": 1153, "group": 144}
-{"id":730, "a": 1458, "group": 182}
-{"id":450, "a": 898, "group": 112}
-{"id":391, "a": 779, "group": 97}
-{"id":256, "a": 512, "group": 64}
-{"id":544, "a": 1088, "group": 136}
-{"id":629, "a": 1257, "group": 157}
-{"id":189, "a": 377, "group": 47}
-{"id":304, "a": 608, "group": 76}
-{"id":508, "a": 1016, "group": 127}
-{"id":681, "a": 1361, "group": 170}
-{"id":86, "a": 170, "group": 21}
-{"id":901, "a": 1801, "group": 225}
-{"id":55, "a": 107, "group": 13}
-{"id":647, "a": 1291, "group": 161}
-{"id":737, "a": 1473, "group": 184}
-{"id":5, "a": 9, "group": 1}
-{"id":879, "a": 1755, "group": 219}
-{"id":913, "a": 1825, "group": 228}
-{"id":557, "a": 1113, "group": 139}
-{"id":430, "a": 858, "group": 107}
-{"id":30, "a": 58, "group": 7}
-{"id":779, "a": 1555, "group": 194}
-{"id":237, "a": 473, "group": 59}
-{"id":238, "a": 474, "group": 59}
-{"id":69, "a": 137, "group": 17}
-{"id":165, "a": 329, "group": 41}
-{"id":804, "a": 1608, "group": 201}
-{"id":672, "a": 1344, "group": 168}
-{"id":904, "a": 1808, "group": 226}
-{"id":20, "a": 40, "group": 5}
-{"id":650, "a": 1298, "group": 162}
-{"id":124, "a": 248, "group": 31}
-{"id":819, "a": 1635, "group": 204}
-{"id":76, "a": 152, "group": 19}
-{"id":918, "a": 1834, "group": 229}
-{"id":855, "a": 1707, "group": 213}
-{"id":922, "a": 1842, "group": 230}
-{"id":562, "a": 1122, "group": 140}
-{"id":101, "a": 201, "group": 25}
-{"id":96, "a": 192, "group": 24}
-{"id":357, "a": 713, "group": 89}
-{"id":279, "a": 555, "group": 69}
-{"id":759, "a": 1515, "group": 189}
-{"id":293, "a": 585, "group": 73}
-{"id":653, "a": 1305, "group": 163}
-{"id":108, "a": 216, "group": 27}
-{"id":254, "a": 506, "group": 63}
-{"id":655, "a": 1307, "group": 163}
-{"id":945, "a": 1889, "group": 236}
-{"id":572, "a": 1144, "group": 143}
-{"id":756, "a": 1512, "group": 189}
-{"id":822, "a": 1642, "group": 205}
-{"id":288, "a": 576, "group": 72}
-{"id":641, "a": 1281, "group": 160}
-{"id":275, "a": 547, "group": 68}
-{"id":654, "a": 1306, "group": 163}
-{"id":896, "a": 1792, "group": 224}
-{"id":192, "a": 384, "group": 48}
-{"id":885, "a": 1769, "group": 221}
-{"id":660, "a": 1320, "group": 165}
-{"id":573, "a": 1145, "group": 143}
-{"id":163, "a": 323, "group": 40}
-{"id":802, "a": 1602, "group": 200}
-{"id":874, "a": 1746, "group": 218}
-{"id":791, "a": 1579, "group": 197}
-{"id":303, "a": 603, "group": 75}
-{"id":267, "a": 531, "group": 66}
-{"id":529, "a": 1057, "group": 132}
-{"id":811, "a": 1619, "group": 202}
-{"id":713, "a": 1425, "group": 178}
-{"id":193, "a": 385, "group": 48}
-{"id":886, "a": 1770, "group": 221}
-{"id":416, "a": 832, "group": 104}
-{"id":786, "a": 1570, "group": 196}
-{"id":15, "a": 27, "group": 3}
-{"id":626, "a": 1250, "group": 156}
-{"id":83, "a": 163, "group": 20}
-{"id":231, "a": 459, "group": 57}
-{"id":777, "a": 1553, "group": 194}
-{"id":78, "a": 154, "group": 19}
-{"id":877, "a": 1753, "group": 219}
-{"id":232, "a": 464, "group": 58}
-{"id":607, "a": 1211, "group": 151}
-{"id":525, "a": 1049, "group": 131}
-{"id":322, "a": 642, "group": 80}
-{"id":41, "a": 81, "group": 10}
-{"id":882, "a": 1762, "group": 220}
-{"id":957, "a": 1913, "group": 239}
-{"id":21, "a": 41, "group": 5}
-{"id":728, "a": 1456, "group": 182}
-{"id":206, "a": 410, "group": 51}
-{"id":775, "a": 1547, "group": 193}
-{"id":2, "a": 2, "group": 0}
-{"id":673, "a": 1345, "group": 168}
-{"id":64, "a": 128, "group": 16}
-{"id":309, "a": 617, "group": 77}
-{"id":415, "a": 827, "group": 103}
-{"id":537, "a": 1073, "group": 134}
-{"id":597, "a": 1193, "group": 149}
-{"id":458, "a": 914, "group": 114}
-{"id":872, "a": 1744, "group": 218}
-{"id":355, "a": 707, "group": 88}
-{"id":638, "a": 1274, "group": 159}
-{"id":546, "a": 1090, "group": 136}
-{"id":140, "a": 280, "group": 35}
-{"id":331, "a": 659, "group": 82}
-{"id":697, "a": 1393, "group": 174}
-{"id":9, "a": 17, "group": 2}
-{"id":60, "a": 120, "group": 15}
-{"id":849, "a": 1697, "group": 212}
-{"id":119, "a": 235, "group": 29}
-{"id":316, "a": 632, "group": 79}
-{"id":782, "a": 1562, "group": 195}
-{"id":565, "a": 1129, "group": 141}
-{"id":494, "a": 986, "group": 123}
-{"id":437, "a": 873, "group": 109}
-{"id":856, "a": 1712, "group": 214}
-{"id":397, "a": 793, "group": 99}
-{"id":742, "a": 1482, "group": 185}
-{"id":692, "a": 1384, "group": 173}
-{"id":854, "a": 1706, "group": 213}
-{"id":68, "a": 136, "group": 17}
-{"id":869, "a": 1737, "group": 217}
-{"id":280, "a": 560, "group": 70}
-{"id":242, "a": 482, "group": 60}
-{"id":66, "a": 130, "group": 16}
-{"id":823, "a": 1643, "group": 205}
-{"id":964, "a": 1928, "group": 241}
-{"id":158, "a": 314, "group": 39}
-{"id":690, "a": 1378, "group": 172}
-{"id":185, "a": 369, "group": 46}
-{"id":619, "a": 1235, "group": 154}
-{"id":400, "a": 800, "group": 100}
-{"id":908, "a": 1816, "group": 227}
-{"id":109, "a": 217, "group": 27}
-{"id":54, "a": 106, "group": 13}
-{"id":511, "a": 1019, "group": 127}
-{"id":111, "a": 219, "group": 27}
-{"id":125, "a": 249, "group": 31}
-{"id":85, "a": 169, "group": 21}
-{"id":617, "a": 1233, "group": 154}
-{"id":798, "a": 1594, "group": 199}
-{"id":399, "a": 795, "group": 99}
-{"id":470, "a": 938, "group": 117}
-{"id":645, "a": 1289, "group": 161}
-{"id":187, "a": 371, "group": 46}
-{"id":474, "a": 946, "group": 118}
-{"id":134, "a": 266, "group": 33}
-{"id":335, "a": 667, "group": 83}
-{"id":711, "a": 1419, "group": 177}
-{"id":145, "a": 289, "group": 36}
-{"id":157, "a": 313, "group": 39}
-{"id":177, "a": 353, "group": 44}
-{"id":808, "a": 1616, "group": 202}
-{"id":662, "a": 1322, "group": 165}
-{"id":420, "a": 840, "group": 105}
-{"id":568, "a": 1136, "group": 142}
-{"id":130, "a": 258, "group": 32}
-{"id":864, "a": 1728, "group": 216}
-{"id":542, "a": 1082, "group": 135}
-{"id":89, "a": 177, "group": 22}
-{"id":26, "a": 50, "group": 6}
-{"id":969, "a": 1937, "group": 242}
-{"id":366, "a": 730, "group": 91}
-{"id":575, "a": 1147, "group": 143}
-{"id":368, "a": 736, "group": 92}
-{"id":308, "a": 616, "group": 77}
-{"id":941, "a": 1881, "group": 235}
-{"id":590, "a": 1178, "group": 147}
-{"id":825, "a": 1649, "group": 206}
-{"id":732, "a": 1464, "group": 183}
-{"id":569, "a": 1137, "group": 142}
-{"id":601, "a": 1201, "group": 150}
-{"id":746, "a": 1490, "group": 186}
-{"id":246, "a": 490, "group": 61}
-{"id":116, "a": 232, "group": 29}
-{"id":873, "a": 1745, "group": 218}
-{"id":181, "a": 361, "group": 45}
-{"id":876, "a": 1752, "group": 219}
-{"id":632, "a": 1264, "group": 158}
-{"id":336, "a": 672, "group": 84}
-{"id":128, "a": 256, "group": 32}
-{"id":292, "a": 584, "group": 73}
-{"id":205, "a": 409, "group": 51}
-{"id":429, "a": 857, "group": 107}
-{"id":845, "a": 1689, "group": 211}
-{"id":143, "a": 283, "group": 35}
-{"id":668, "a": 1336, "group": 167}
-{"id":744, "a": 1488, "group": 186}
-{"id":630, "a": 1258, "group": 157}
-{"id":53, "a": 105, "group": 13}
-{"id":792, "a": 1584, "group": 198}
-{"id":593, "a": 1185, "group": 148}
-{"id":803, "a": 1603, "group": 200}
-{"id":899, "a": 1795, "group": 224}
-{"id":253, "a": 505, "group": 63}
-{"id":243, "a": 483, "group": 60}
-{"id":245, "a": 489, "group": 61}
-{"id":82, "a": 162, "group": 20}
-{"id":497, "a": 993, "group": 124}
-{"id":95, "a": 187, "group": 23}
-{"id":7, "a": 11, "group": 1}
-{"id":919, "a": 1835, "group": 229}
-{"id":710, "a": 1418, "group": 177}
-{"id":351, "a": 699, "group": 87}
-{"id":651, "a": 1299, "group": 162}
-{"id":954, "a": 1906, "group": 238}
-{"id":170, "a": 338, "group": 42}
-{"id":32, "a": 64, "group": 8}
-{"id":222, "a": 442, "group": 55}
-{"id":479, "a": 955, "group": 119}
-{"id":706, "a": 1410, "group": 176}
-{"id":564, "a": 1128, "group": 141}
-{"id":769, "a": 1537, "group": 192}
-{"id":524, "a": 1048, "group": 131}
-{"id":46, "a": 90, "group": 11}
-{"id":793, "a": 1585, "group": 198}
-{"id":837, "a": 1673, "group": 209}
-{"id":979, "a": 1955, "group": 244}
-{"id":962, "a": 1922, "group": 240}
-{"id":740, "a": 1480, "group": 185}
-{"id":282, "a": 562, "group": 70}
-{"id":724, "a": 1448, "group": 181}
-{"id":509, "a": 1017, "group": 127}
-{"id":266, "a": 530, "group": 66}
-{"id":271, "a": 539, "group": 67}
-{"id":155, "a": 307, "group": 38}
-{"id":18, "a": 34, "group": 4}
-{"id":339, "a": 675, "group": 84}
-{"id":598, "a": 1194, "group": 149}
-{"id":948, "a": 1896, "group": 237}
-{"id":910, "a": 1818, "group": 227}
-{"id":846, "a": 1690, "group": 211}
-{"id":138, "a": 274, "group": 34}
-{"id":760, "a": 1520, "group": 190}
-{"id":853, "a": 1705, "group": 213}
-{"id":860, "a": 1720, "group": 215}
-{"id":283, "a": 563, "group": 70}
-{"id":188, "a": 376, "group": 47}
-{"id":613, "a": 1225, "group": 153}
-{"id":321, "a": 641, "group": 80}
-{"id":148, "a": 296, "group": 37}
-{"id":674, "a": 1346, "group": 168}
-{"id":226, "a": 450, "group": 56}
-{"id":994, "a": 1986, "group": 248}
-{"id":998, "a": 1994, "group": 249}
-{"id":97, "a": 193, "group": 24}
-{"id":160, "a": 320, "group": 40}
-{"id":616, "a": 1232, "group": 154}
-{"id":424, "a": 848, "group": 106}
-{"id":937, "a": 1873, "group": 234}
-{"id":392, "a": 784, "group": 98}
-{"id":25, "a": 49, "group": 6}
-{"id":88, "a": 176, "group": 22}
-{"id":534, "a": 1066, "group": 133}
-{"id":536, "a": 1072, "group": 134}
-{"id":530, "a": 1058, "group": 132}
-{"id":289, "a": 577, "group": 72}
-{"id":861, "a": 1721, "group": 215}
-{"id":376, "a": 752, "group": 94}
-{"id":682, "a": 1362, "group": 170}
-{"id":327, "a": 651, "group": 81}
-{"id":566, "a": 1130, "group": 141}
-{"id":199, "a": 395, "group": 49}
-{"id":902, "a": 1802, "group": 225}
-{"id":433, "a": 865, "group": 108}
-{"id":13, "a": 25, "group": 3}
-{"id":103, "a": 203, "group": 25}
-{"id":907, "a": 1811, "group": 226}
-{"id":989, "a": 1977, "group": 247}
-{"id":974, "a": 1946, "group": 243}
-{"id":978, "a": 1954, "group": 244}
-{"id":107, "a": 211, "group": 26}
-{"id":722, "a": 1442, "group": 180}
-{"id":38, "a": 74, "group": 9}
-{"id":549, "a": 1097, "group": 137}
-{"id":485, "a": 969, "group": 121}
-{"id":218, "a": 434, "group": 54}
-{"id":927, "a": 1851, "group": 231}
-{"id":725, "a": 1449, "group": 181}
-{"id":702, "a": 1402, "group": 175}
-{"id":198, "a": 394, "group": 49}
-{"id":214, "a": 426, "group": 53}
-{"id":373, "a": 745, "group": 93}
-{"id":147, "a": 291, "group": 36}
-{"id":63, "a": 123, "group": 15}
-{"id":79, "a": 155, "group": 19}
-{"id":543, "a": 1083, "group": 135}
-{"id":334, "a": 666, "group": 83}
-{"id":59, "a": 115, "group": 14}
-{"id":459, "a": 915, "group": 114}
-{"id":495, "a": 987, "group": 123}
-{"id":211, "a": 419, "group": 52}
-{"id":554, "a": 1106, "group": 138}
-{"id":62, "a": 122, "group": 15}
-{"id":758, "a": 1514, "group": 189}
-{"id":272, "a": 544, "group": 68}
-{"id":727, "a": 1451, "group": 181}
-{"id":868, "a": 1736, "group": 217}
-{"id":631, "a": 1259, "group": 157}
-{"id":152, "a": 304, "group": 38}
-{"id":669, "a": 1337, "group": 167}
-{"id":840, "a": 1680, "group": 210}
-{"id":983, "a": 1963, "group": 245}
-{"id":595, "a": 1187, "group": 148}
-{"id":685, "a": 1369, "group": 171}
-{"id":441, "a": 881, "group": 110}
-{"id":527, "a": 1051, "group": 131}
-{"id":820, "a": 1640, "group": 205}
-{"id":434, "a": 866, "group": 108}
-{"id":277, "a": 553, "group": 69}
-{"id":90, "a": 178, "group": 22}
-{"id":676, "a": 1352, "group": 169}
-{"id":627, "a": 1251, "group": 156}
-{"id":614, "a": 1226, "group": 153}
-{"id":432, "a": 864, "group": 108}
-{"id":350, "a": 698, "group": 87}
-{"id":247, "a": 491, "group": 61}
-{"id":551, "a": 1099, "group": 137}
-{"id":29, "a": 57, "group": 7}
-{"id":104, "a": 208, "group": 26}
-{"id":801, "a": 1601, "group": 200}
-{"id":589, "a": 1177, "group": 147}
-{"id":409, "a": 817, "group": 102}
-{"id":248, "a": 496, "group": 62}
-{"id":39, "a": 75, "group": 9}
-{"id":953, "a": 1905, "group": 238}
-{"id":438, "a": 874, "group": 109}
-{"id":278, "a": 554, "group": 69}
-{"id":447, "a": 891, "group": 111}
-{"id":16, "a": 32, "group": 4}
-{"id":426, "a": 850, "group": 106}
-{"id":950, "a": 1898, "group": 237}
-{"id":533, "a": 1065, "group": 133}
-{"id":171, "a": 339, "group": 42}
-{"id":482, "a": 962, "group": 120}
-{"id":933, "a": 1865, "group": 233}
-{"id":701, "a": 1401, "group": 175}
-{"id":428, "a": 856, "group": 107}
-{"id":915, "a": 1827, "group": 228}
-{"id":972, "a": 1944, "group": 243}
-{"id":857, "a": 1713, "group": 214}
-{"id":844, "a": 1688, "group": 211}
-{"id":683, "a": 1363, "group": 170}
-{"id":3, "a": 3, "group": 0}
-{"id":65, "a": 129, "group": 16}
-{"id":121, "a": 241, "group": 30}
-{"id":202, "a": 402, "group": 50}
-{"id":753, "a": 1505, "group": 188}
-{"id":369, "a": 737, "group": 92}
-{"id":765, "a": 1529, "group": 191}
-{"id":661, "a": 1321, "group": 165}
-{"id":342, "a": 682, "group": 85}
-{"id":442, "a": 882, "group": 110}
-{"id":592, "a": 1184, "group": 148}
-{"id":717, "a": 1433, "group": 179}
-{"id":608, "a": 1216, "group": 152}
-{"id":72, "a": 144, "group": 18}
-{"id":698, "a": 1394, "group": 174}
-{"id":560, "a": 1120, "group": 140}
-{"id":809, "a": 1617, "group": 202}
-{"id":721, "a": 1441, "group": 180}
-{"id":176, "a": 352, "group": 44}
-{"id":87, "a": 171, "group": 21}
-{"id":10, "a": 18, "group": 2}
-{"id":414, "a": 826, "group": 103}
-{"id":548, "a": 1096, "group": 137}
-{"id":37, "a": 73, "group": 9}
-{"id":183, "a": 363, "group": 45}
-{"id":559, "a": 1115, "group": 139}
-{"id":736, "a": 1472, "group": 184}
-{"id":146, "a": 290, "group": 36}
-{"id":776, "a": 1552, "group": 194}
-{"id":505, "a": 1009, "group": 126}
-{"id":993, "a": 1985, "group": 248}
-{"id":992, "a": 1984, "group": 248}
-{"id":312, "a": 624, "group": 78}
-{"id":166, "a": 330, "group": 41}
-{"id":696, "a": 1392, "group": 174}
-{"id":27, "a": 51, "group": 6}
-{"id":269, "a": 537, "group": 67}
-{"id":139, "a": 275, "group": 34}
-{"id":504, "a": 1008, "group": 126}
-{"id":43, "a": 83, "group": 10}
-{"id":469, "a": 937, "group": 117}
-{"id":832, "a": 1664, "group": 208}
-{"id":380, "a": 760, "group": 95}
-{"id":168, "a": 336, "group": 42}
-{"id":768, "a": 1536, "group": 192}
-{"id":42, "a": 82, "group": 10}
-{"id":489, "a": 977, "group": 122}
-{"id":396, "a": 792, "group": 99}
-{"id":584, "a": 1168, "group": 146}
-{"id":975, "a": 1947, "group": 243}
-{"id":359, "a": 715, "group": 89}
-{"id":220, "a": 440, "group": 55}
-{"id":797, "a": 1593, "group": 199}
-{"id":298, "a": 594, "group": 74}
-{"id":486, "a": 970, "group": 121}
-{"id":997, "a": 1993, "group": 249}
-{"id":790, "a": 1578, "group": 197}
-{"id":453, "a": 905, "group": 113}
-{"id":735, "a": 1467, "group": 183}
-{"id":24, "a": 48, "group": 6}
-{"id":502, "a": 1002, "group": 125}
-{"id":939, "a": 1875, "group": 234}
-{"id":314, "a": 626, "group": 78}
-{"id":457, "a": 913, "group": 114}
-{"id":132, "a": 264, "group": 33}
-{"id":50, "a": 98, "group": 12}
-{"id":454, "a": 906, "group": 113}
-{"id":576, "a": 1152, "group": 144}
-{"id":881, "a": 1761, "group": 220}
-{"id":633, "a": 1265, "group": 158}
-{"id":353, "a": 705, "group": 88}
-{"id":934, "a": 1866, "group": 233}
-{"id":620, "a": 1240, "group": 155}
-{"id":167, "a": 331, "group": 41}
-{"id":579, "a": 1155, "group": 144}
-{"id":99, "a": 195, "group": 24}
-{"id":799, "a": 1595, "group": 199}
-{"id":762, "a": 1522, "group": 190}
-{"id":547, "a": 1091, "group": 136}
-{"id":100, "a": 200, "group": 25}
-{"id":154, "a": 306, "group": 38}
-{"id":778, "a": 1554, "group": 194}
-{"id":151, "a": 299, "group": 37}
-{"id":570, "a": 1138, "group": 142}
-{"id":888, "a": 1776, "group": 222}
-{"id":665, "a": 1329, "group": 166}
-{"id":44, "a": 88, "group": 11}
-{"id":670, "a": 1338, "group": 167}
-{"id":412, "a": 824, "group": 103}
-{"id":984, "a": 1968, "group": 246}
-{"id":659, "a": 1315, "group": 164}
-{"id":307, "a": 611, "group": 76}
-{"id":197, "a": 393, "group": 49}
-{"id":306, "a": 610, "group": 76}
-{"id":468, "a": 936, "group": 117}
-{"id":262, "a": 522, "group": 65}
-{"id":483, "a": 963, "group": 120}
-{"id":610, "a": 1218, "group": 152}
-{"id":507, "a": 1011, "group": 126}
-{"id":609, "a": 1217, "group": 152}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/oneKeyCount.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/oneKeyCount.json b/exec/java-exec/src/test/resources/window/oneKeyCount.json
deleted file mode 100644
index fa5cd8c..0000000
--- a/exec/java-exec/src/test/resources/window/oneKeyCount.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_PHYSICAL",
- version:"1",
- generator:{
- type:"manual"
- }
- },
- graph:[
- {
- @id:1,
- pop:"fs-scan",
- format: {type: "json"},
- storage:{type: "file", connection: "file:///"},
- files:["#{DATA_FILE}"]
- },
- {
- @id:2,
- child: 1,
- pop:"external-sort",
- orderings: [
- {expr: "a"}
- ]
- },
- {
- @id:3,
- child: 2,
- pop:"window",
- within: [
- { ref: "a", expr: "a" }
- ],
- aggregations: [
- { ref: "cnt", expr:"count(1)" },
- { ref: "sum", expr:"sum(b)" }
- ]
- },
- {
- @id: 4,
- child: 3,
- pop: "screen"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/oneKeyCountData.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/oneKeyCountData.json b/exec/java-exec/src/test/resources/window/oneKeyCountData.json
deleted file mode 100644
index 3c0115e..0000000
--- a/exec/java-exec/src/test/resources/window/oneKeyCountData.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{"a": 1, "b": 100}
- {"a": 1, "b": 50}
- {"a": 2, "b": 25}
- {"a": 2, "b": 50}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/oneKeyCountMultiBatch.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/oneKeyCountMultiBatch.json b/exec/java-exec/src/test/resources/window/oneKeyCountMultiBatch.json
deleted file mode 100644
index 09a405c..0000000
--- a/exec/java-exec/src/test/resources/window/oneKeyCountMultiBatch.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
- "head" : {
- "version" : 1,
- "generator" : {
- "type" : "DefaultSqlHandler",
- "info" : ""
- },
- "type" : "APACHE_DRILL_PHYSICAL",
- "options" : [ ],
- "resultMode" : "EXEC"
- },
- "graph" : [{
- @id:1,
- pop:"fs-scan",
- format: {type: "json"},
- storage:{type: "file", connection: "file:///"},
- files:["#{DATA_FILE}"]
- },
- {
- @id:2,
- child: 1,
- pop:"external-sort",
- orderings: [
- {expr: "group"},
- {expr: "a"}
- ]
- },
- {
- "pop" : "window",
- "@id" : 3,
- "child" : 2,
- "aggregations" : [ {
- "ref" : "`w0$o0`",
- "expr" : "count(`a`) "
- }, {
- "ref" : "`w0$o1`",
- "expr" : "$sum0(`a`) "
- } ],
- "start" : -9223372036854775808,
- "end" : -9223372036854775808,
- "initialAllocation" : 1000000,
- "maxAllocation" : 10000000000,
- "withins" : [ {
- "ref" : "`group`",
- "expr" : "`group`"
- } ]
- }, {
- "pop" : "project",
- "@id" : 4,
- "exprs" : [ {
- "ref" : "`output`",
- "expr" : " ( if (greater_than(`w0$o0`, 0) ) then (`w0$o1` ) else (NULL) end ) "
- },
- {
- "ref" : "cnt",
- "expr": "w0$o0"
- },
- {
- "ref" : "sum",
- "expr": "w0$o1"
- }],
- "child" : 3,
- "initialAllocation" : 1000000,
- "maxAllocation" : 10000000000
- }, {
- "pop" : "screen",
- "@id" : 5,
- "child" : 4,
- "initialAllocation" : 1000000,
- "maxAllocation" : 10000000000
- } ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/twoKeys.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/twoKeys.json b/exec/java-exec/src/test/resources/window/twoKeys.json
deleted file mode 100644
index f3ef4a5..0000000
--- a/exec/java-exec/src/test/resources/window/twoKeys.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- head:{
- type:"APACHE_DRILL_PHYSICAL",
- version:"1",
- generator:{
- type:"manual"
- }
- },
- graph:[
- {
- @id:1,
- pop:"fs-scan",
- format: {type: "json"},
- storage:{type: "file", connection: "file:///"},
- files:["#{DATA_FILE}"]
- },
- {
- @id:2,
- child: 1,
- pop:"external-sort",
- orderings: [
- {expr: "a"}
- ]
- },
- {
- @id:3,
- child: 2,
- pop:"window",
- within: [
- { ref: "a", expr: "a" },
- { ref: "b", expr: "b" }
- ],
- aggregations: [
- { ref: "cnt", expr:"count(1)" },
- { ref: "sum", expr:"sum(c)" }
- ]
- },
- {
- @id: 4,
- child: 3,
- pop: "screen"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/twoKeysData.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/twoKeysData.json b/exec/java-exec/src/test/resources/window/twoKeysData.json
deleted file mode 100644
index fd09236..0000000
--- a/exec/java-exec/src/test/resources/window/twoKeysData.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{"a": 1, "b": "group1", "c": 5}
-{"a": 1, "b": "group1", "c": 10}
-{"a": 1, "b": "group2", "c": 15}
-{"a": 1, "b": "group2", "c": 20}
-{"a": 2, "b": "group3", "c": 25}
-{"a": 2, "b": "group3", "c": 30}
-{"a": 2, "b": "group4", "c": 35}
-{"a": 2, "b": "group4", "c": 40}
\ No newline at end of file
[2/2] drill git commit: DRILL-3200: Add Window functions: ROW_NUMBER,
RANK, PERCENT_RANK, DENSE_RANK and CUME_DIST
Posted by jn...@apache.org.
DRILL-3200: Add Window functions: ROW_NUMBER, RANK, PERCENT_RANK, DENSE_RANK and CUME_DIST
- enum WindowFrameRecordBatch.WindowFunction to handle supported window function and their corresponding output MajorType
- renamed WindowFrameTemplate -> DefaultFrameTemplate, cleaned the template to handle the default frame efficiently:
. a batch can be processed as soon as we find the last peer row of it's last row
. once a batch is processed it can be safely released => we can transfer it's value vectors to the container instead of copying them
- DefaultFrameTemplate.Partition tracks the current window frame and computes the following window functions automatically: row_number, rank, dense_rank, percent_rank, cume_dist. It doesn't need to aggregate the value vectors to compute these window functions
- updated TestWindowFrame to check the results of row_number, rank, dense_rank, percent_rank and cume_dist in various cases
. added a debug config option to MSorter to control the size of batches. This is needed by TestWindowFrame so it can use small test data files (20 rows per batch)
. removed contrib/data/window-test-data
- WindowFrameRecordBatch properly releases saved batches if the query stops prematurely
- GenerateTestData can be used to generate test data for the window function unit tests [it's a work in progress and can be either improved to make it developer friendly or removed from the final patch]
- using newly created WindowDataBatch in place of RecordDataBatch, to expose FragmentContext and VectorAccessible (fixes DRILL-3218)
- window.enable is true by default
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3bccec91
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3bccec91
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3bccec91
Branch: refs/heads/master
Commit: 3bccec9110c7ff86fa3cf04baa81a1747e1f5b9e
Parents: 453f6f7
Author: adeneche <ad...@gmail.com>
Authored: Fri Mar 13 10:06:32 2015 -0700
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Jun 11 10:59:35 2015 -0700
----------------------------------------------------------------------
contrib/data/pom.xml | 1 -
contrib/data/window-test-data/pom.xml | 64 --
exec/java-exec/pom.xml | 6 -
.../org/apache/drill/exec/ExecConstants.java | 3 +-
.../impl/window/DefaultFrameTemplate.java | 320 ++++++
.../exec/physical/impl/window/Partition.java | 82 ++
.../physical/impl/window/WindowDataBatch.java | 93 ++
.../impl/window/WindowFrameRecordBatch.java | 216 ++--
.../impl/window/WindowFrameTemplate.java | 379 -------
.../exec/physical/impl/window/WindowFramer.java | 21 +-
.../exec/physical/impl/xsort/MSortTemplate.java | 15 +-
.../drill/exec/record/AbstractRecordBatch.java | 6 +-
.../apache/drill/TestDisabledFunctionality.java | 13 -
.../physical/impl/window/GenerateTestData.java | 286 +++++
.../physical/impl/window/TestWindowFrame.java | 91 +-
.../src/test/resources/window/allData.csv | 5 +
.../src/test/resources/window/b1.p1.subs.tsv | 20 +
.../src/test/resources/window/b1.p1.tsv | 20 +
.../src/test/resources/window/b1.p1/0.data.json | 21 +
.../src/test/resources/window/b1.p2.subs.tsv | 20 +
.../src/test/resources/window/b1.p2.tsv | 20 +
.../src/test/resources/window/b1.p2/0.data.json | 22 +
.../src/test/resources/window/b2.p2.subs.tsv | 40 +
.../src/test/resources/window/b2.p2.tsv | 40 +
.../src/test/resources/window/b2.p2/0.data.json | 22 +
.../src/test/resources/window/b2.p2/1.data.json | 20 +
.../src/test/resources/window/b2.p4.subs.tsv | 40 +
.../src/test/resources/window/b2.p4.tsv | 40 +
.../src/test/resources/window/b2.p4/0.data.json | 24 +
.../src/test/resources/window/b2.p4/1.data.json | 20 +
.../src/test/resources/window/b3.p2.subs.tsv | 60 ++
.../src/test/resources/window/b3.p2.tsv | 60 ++
.../src/test/resources/window/b3.p2/0.data.json | 22 +
.../src/test/resources/window/b3.p2/1.data.json | 20 +
.../src/test/resources/window/b3.p2/2.data.json | 20 +
.../src/test/resources/window/b4.p4.subs.tsv | 80 ++
.../src/test/resources/window/b4.p4.tsv | 80 ++
.../src/test/resources/window/b4.p4/0.data.json | 24 +
.../src/test/resources/window/b4.p4/1.data.json | 20 +
.../src/test/resources/window/b4.p4/2.data.json | 20 +
.../src/test/resources/window/b4.p4/3.data.json | 20 +
.../src/test/resources/window/mediumData.json | 1000 ------------------
.../src/test/resources/window/oneKeyCount.json | 43 -
.../test/resources/window/oneKeyCountData.json | 4 -
.../resources/window/oneKeyCountMultiBatch.json | 72 --
.../src/test/resources/window/twoKeys.json | 44 -
.../src/test/resources/window/twoKeysData.json | 8 -
47 files changed, 1812 insertions(+), 1755 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/contrib/data/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/data/pom.xml b/contrib/data/pom.xml
index d1def76..450bc0d 100644
--- a/contrib/data/pom.xml
+++ b/contrib/data/pom.xml
@@ -33,6 +33,5 @@
<modules>
<module>tpch-sample-data</module>
- <module>window-test-data</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/contrib/data/window-test-data/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/data/window-test-data/pom.xml b/contrib/data/window-test-data/pom.xml
deleted file mode 100644
index 6d195da..0000000
--- a/contrib/data/window-test-data/pom.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- You under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>drill-contrib-data-parent</artifactId>
- <groupId>org.apache.drill.contrib.data</groupId>
- <version>1.1.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>window-test-data</artifactId>
- <name>contrib/data/window-test-data</name>
- <packaging>jar</packaging>
-
- <dependencies>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.0</version>
- <executions>
- <execution>
- <id>install-tgz</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>https://s3-us-west-2.amazonaws.com/denbucket/window_test_data_0.1.tgz</url>
- <outputFileName>window.tgz</outputFileName>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/classes/window</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <pluginRepositories>
- <pluginRepository>
- <id>sonatype-public-repository</id>
- <url>https://oss.sonatype.org/content/groups/public</url>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- <releases>
- <enabled>true</enabled>
- </releases>
- </pluginRepository>
- </pluginRepositories>
-
-</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index b5cd52b..5cc209d 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -38,12 +38,6 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.drill.contrib.data</groupId>
- <artifactId>window-test-data</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
<!-- <dependency> -->
<!-- <groupId>org.ow2.asm</groupId> -->
<!-- <artifactId>asm-commons</artifactId> -->
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 91793f5..8ea90e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -71,6 +71,7 @@ public interface ExecConstants {
public static final String EXTERNAL_SORT_SPILL_THRESHOLD = "drill.exec.sort.external.spill.threshold";
public static final String EXTERNAL_SORT_SPILL_DIRS = "drill.exec.sort.external.spill.directories";
public static final String EXTERNAL_SORT_SPILL_FILESYSTEM = "drill.exec.sort.external.spill.fs";
+ public static final String EXTERNAL_SORT_MSORT_MAX_BATCHSIZE = "drill.exec.sort.external.msort.batch.maxsize";
public static final String TEXT_LINE_READER_BATCH_SIZE = "drill.exec.storage.file.text.batch.size";
public static final String TEXT_LINE_READER_BUFFER_SIZE = "drill.exec.storage.file.text.buffer.size";
public static final String HAZELCAST_SUBNETS = "drill.exec.cache.hazel.subnets";
@@ -238,7 +239,7 @@ public interface ExecConstants {
public static final String DRILL_SYS_FILE_SUFFIX = ".sys.drill";
public static final String ENABLE_WINDOW_FUNCTIONS = "window.enable";
- public static final OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS, false);
+ public static final OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS, true);
public static final String DRILLBIT_CONTROL_INJECTIONS = "drill.exec.testing.controls";
public static final OptionValidator DRILLBIT_CONTROLS_VALIDATOR =
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
new file mode 100644
index 0000000..ada068b
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.window;
+
+import org.apache.drill.common.exceptions.DrillException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
+
+import javax.inject.Named;
+import java.util.Iterator;
+import java.util.List;
+
+
+public abstract class DefaultFrameTemplate implements WindowFramer {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DefaultFrameTemplate.class);
+
+ private VectorContainer container;
+ private List<WindowDataBatch> batches;
+ private int outputCount; // number of rows in currently/last processed batch
+
+ /**
+ * current partition being processed.</p>
+ * Can span over multiple batches, so we may need to keep it between calls to doWork()
+ */
+ private Partition partition;
+
+ @Override
+ public void setup(List<WindowDataBatch> batches, final VectorContainer container) throws SchemaChangeException {
+ this.container = container;
+ this.batches = batches;
+
+ outputCount = 0;
+ partition = null;
+ }
+
+ private void allocateOutgoing() {
+ for (VectorWrapper<?> w : container) {
+ w.getValueVector().allocateNew();
+ }
+ }
+
+ /**
+ * processes all rows of current batch:
+ * <ul>
+ * <li>compute aggregations</li>
+ * <li>compute window functions</li>
+ * <li>transfer remaining vectors from current batch to container</li>
+ * </ul>
+ */
+ @Override
+ public void doWork() throws DrillException {
+ int currentRow = 0;
+
+ logger.trace("WindowFramer.doWork() START, num batches {}, current batch has {} rows",
+ batches.size(), batches.get(0).getRecordCount());
+
+ allocateOutgoing();
+
+ final WindowDataBatch current = batches.get(0);
+
+ // we need to store the record count explicitly, because we release current batch at the end of this call
+ outputCount = current.getRecordCount();
+
+ while (currentRow < outputCount) {
+ if (partition != null) {
+ assert currentRow == 0 : "pending windows are only expected at the start of the batch";
+
+ // we have a pending window we need to handle from a previous call to doWork()
+ logger.trace("we have a pending partition {}", partition);
+ } else {
+ final int length = computePartitionSize(currentRow);
+ partition = new Partition(length);
+ setupWrite(current, container);
+ }
+
+ currentRow = processPartition(currentRow);
+ if (partition.isDone()) {
+ partition = null;
+ resetValues();
+ }
+ }
+
+ // transfer "non aggregated" vectors
+ for (VectorWrapper<?> vw : current) {
+ ValueVector v = container.addOrGet(vw.getField());
+ TransferPair tp = vw.getValueVector().makeTransferPair(v);
+ tp.transfer();
+ }
+
+ for (VectorWrapper<?> v : container) {
+ v.getValueVector().getMutator().setValueCount(outputCount);
+ }
+
+ // because we are using the default frame, and we keep the aggregated value until we start a new frame
+ // we can safely free the current batch
+ batches.remove(0).clear();
+
+ logger.trace("WindowFramer.doWork() END");
+ }
+
+ /**
+ * process all rows (computes and writes aggregation values) of current batch that are part of current partition.
+ * @param currentRow first unprocessed row
+ * @return index of next unprocessed row
+ * @throws DrillException if it can't write into the container
+ */
+ private int processPartition(final int currentRow) throws DrillException {
+ logger.trace("process partition {}, currentRow: {}, outputCount: {}", partition, currentRow, outputCount);
+
+ int row = currentRow;
+ while (row < outputCount && !partition.isDone()) {
+ if (partition.isFrameDone()) {
+ // because all peer rows share the same frame, we only need to compute and aggregate the frame once
+ partition.newFrame(countPeers(row));
+ aggregatePeers(row);
+ }
+
+ outputAggregatedValues(row, partition);
+
+ partition.rowAggregated();
+ row++;
+ }
+
+ return row;
+ }
+
+ /**
+ * @return number of rows that are part of the partition starting at row start of first batch
+ */
+ private int computePartitionSize(final int start) {
+ logger.trace("compute partition size starting from {} on {} batches", start, batches.size());
+
+ // current partition always starts from first batch
+ final VectorAccessible first = getCurrent();
+
+ int length = 0;
+
+ // count all rows that are in the same partition of start
+ // keep increasing length until we find first row of next partition or we reach the very
+ // last batch
+ for (WindowDataBatch batch : batches) {
+ final int recordCount = batch.getRecordCount();
+
+ // check first container from start row, and subsequent containers from first row
+ for (int row = (batch == first) ? start : 0; row < recordCount; row++, length++) {
+ if (!isSamePartition(start, first, row, batch)) {
+ return length;
+ }
+ }
+ }
+
+ return length;
+ }
+
+ /**
+ * Counts how many rows are peer with the first row of the current frame
+ * @param start first row of current frame
+ * @return number of peer rows
+ */
+ private int countPeers(final int start) {
+ // current frame always starts from first batch
+ final VectorAccessible first = getCurrent();
+
+ int length = 0;
+
+ // count all rows that are in the same frame of starting row
+ // keep increasing length until we find first non peer row we reach the very
+ // last batch
+ for (WindowDataBatch batch : batches) {
+ final int recordCount = batch.getRecordCount();
+
+ // for every remaining row in the partition, count it if it's a peer row
+ final int remaining = partition.getRemaining();
+ for (int row = (batch == first) ? start : 0; row < recordCount && length < remaining; row++, length++) {
+ if (!isPeer(start, first, row, batch)) {
+ return length;
+ }
+ }
+ }
+
+ return length;
+ }
+
+ /**
+ * aggregates all peer rows of current row
+ * @param currentRow starting row of the current frame
+ * @throws SchemaChangeException
+ */
+ private void aggregatePeers(final int currentRow) throws SchemaChangeException {
+ logger.trace("aggregating {} rows starting from {}", partition.getPeers(), currentRow);
+ assert !partition.isFrameDone() : "frame is empty!";
+
+ // a single frame can include rows from multiple batches
+ // start processing first batch and, if necessary, move to next batches
+ Iterator<WindowDataBatch> iterator = batches.iterator();
+ WindowDataBatch current = iterator.next();
+ setupRead(current, container);
+
+ final int peers = partition.getPeers();
+ for (int i = 0, row = currentRow; i < peers; i++, row++) {
+ if (row >= current.getRecordCount()) {
+ // we reached the end of the current batch, move to the next one
+ current = iterator.next();
+ setupRead(current, container);
+ row = 0;
+ }
+
+ aggregateRecord(row);
+ }
+ }
+
+ @Override
+ public boolean canDoWork() {
+ // check if we can process a saved batch
+ if (batches.size() < 2) {
+ logger.trace("we don't have enough batches to proceed, fetch next batch");
+ return false;
+ }
+
+ final VectorAccessible current = getCurrent();
+ final int currentSize = current.getRecordCount();
+ final VectorAccessible last = batches.get(batches.size() - 1);
+ final int lastSize = last.getRecordCount();
+
+ if (!isSamePartition(currentSize - 1, current, lastSize - 1, last)
+ || !isPeer(currentSize - 1, current, lastSize - 1, last)) {
+ logger.trace("frame changed, we are ready to process first saved batch");
+ return true;
+ } else {
+ logger.trace("frame didn't change, fetch next batch");
+ return false;
+ }
+ }
+
+ /**
+ * @return saved batch that will be processed in doWork()
+ */
+ private VectorAccessible getCurrent() {
+ return batches.get(0);
+ }
+
+ @Override
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ @Override
+ public void cleanup() {
+ }
+
+ /**
+ * setup incoming container for aggregateRecord()
+ */
+ public abstract void setupRead(@Named("incoming") VectorAccessible incoming, @Named("outgoing") VectorAccessible outgoing) throws SchemaChangeException;
+
+ /**
+ * setup outgoing container for outputAggregatedValues. This will also reset the aggregations in most cases.
+ */
+ public abstract void setupWrite(@Named("incoming") WindowDataBatch incoming, @Named("outgoing") VectorAccessible outgoing) throws SchemaChangeException;
+
+ /**
+ * aggregates a row from the incoming container
+ * @param index of row to aggregate
+ */
+ public abstract void aggregateRecord(@Named("index") int index);
+
+ /**
+ * writes aggregated values to row of outgoing container
+ * @param outIndex index of row
+ */
+ public abstract void outputAggregatedValues(@Named("outIndex") int outIndex, @Named("partition") Partition partition);
+
+ /**
+ * reset all window functions
+ */
+ public abstract boolean resetValues();
+
+ /**
+ * compares two rows from different batches (can be the same), if they have the same value for the partition by
+ * expression
+ * @param b1Index index of first row
+ * @param b1 batch for first row
+ * @param b2Index index of second row
+ * @param b2 batch for second row
+ * @return true if the rows are in the same partition
+ */
+ public abstract boolean isSamePartition(@Named("b1Index") int b1Index, @Named("b1") VectorAccessible b1,
+ @Named("b2Index") int b2Index, @Named("b2") VectorAccessible b2);
+
+ /**
+ * compares two rows from different batches (can be the same), if they have the same value for the order by
+ * expression
+ * @param b1Index index of first row
+ * @param b1 batch for first row
+ * @param b2Index index of second row
+ * @param b2 batch for second row
+ * @return true if the rows are in the same partition
+ */
+ public abstract boolean isPeer(@Named("b1Index") int b1Index, @Named("b1") VectorAccessible b1,
+ @Named("b2Index") int b2Index, @Named("b2") VectorAccessible b2);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/Partition.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/Partition.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/Partition.java
new file mode 100644
index 0000000..8d6728e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/Partition.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.window;
+
+/**
+ * Used internally to keep track of partitions and frames
+ */
+public class Partition {
+ private final int length; // size of this partition
+ private int remaining;
+ private int peers;
+
+ // we keep these attributes public because the generated code needs to access them
+ public int row_number;
+ public int rank;
+ public int dense_rank;
+ public double percent_rank;
+ public double cume_dist;
+
+ /**
+ * @return number of rows not yet aggregated in this partition
+ */
+ public int getRemaining() {
+ return remaining;
+ }
+
+ /**
+ * @return peer rows not yet aggregated in current frame
+ */
+ public int getPeers() {
+ return peers;
+ }
+
+ public Partition(int length) {
+ this.length = length;
+ remaining = length;
+ row_number = 1;
+ }
+
+ public void rowAggregated() {
+ remaining--;
+ peers--;
+
+ row_number++;
+ }
+
+ public void newFrame(int peers) {
+ this.peers = peers;
+ rank = row_number; // rank = row number of 1st peer
+ dense_rank++;
+ percent_rank = length > 1 ? (double) (rank - 1) / (length - 1) : 0;
+ cume_dist = (double)(rank + peers - 1) / length;
+ }
+
+ public boolean isDone() {
+ return remaining == 0;
+ }
+
+ public boolean isFrameDone() {
+ return peers == 0;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("{length: %d, remaining partition: %d, remaining peers: %d}", length, remaining, peers);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowDataBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowDataBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowDataBatch.java
new file mode 100644
index 0000000..5045cb3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowDataBatch.java
@@ -0,0 +1,93 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.physical.impl.window;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.Iterator;
+import java.util.List;
+
+public class WindowDataBatch implements VectorAccessible {
+
+ private final FragmentContext context;
+ private final VectorContainer container;
+ private final int recordCount;
+
+ public WindowDataBatch(final VectorAccessible batch, final FragmentContext context) {
+ this.context = context;
+ recordCount = batch.getRecordCount();
+
+ List<ValueVector> vectors = Lists.newArrayList();
+
+ for (VectorWrapper<?> v : batch) {
+ if (v.isHyper()) {
+ throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch.");
+ }
+ TransferPair tp = v.getValueVector().getTransferPair();
+ tp.transfer();
+ vectors.add(tp.getTo());
+ }
+
+ container = new VectorContainer();
+ container.addCollection(vectors);
+ container.setRecordCount(recordCount);
+ container.buildSchema(batch.getSchema().getSelectionVectorMode());
+ }
+
+ public FragmentContext getContext() {
+ return context;
+ }
+
+ @Override
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ @Override
+ public VectorWrapper<?> getValueAccessorById(Class<?> clazz, int... fieldIds) {
+ return container.getValueAccessorById(clazz, fieldIds);
+ }
+
+ @Override
+ public TypedFieldId getValueVectorId(SchemaPath path) {
+ return container.getValueVectorId(path);
+ }
+
+ @Override
+ public BatchSchema getSchema() {
+ return container.getSchema();
+ }
+
+ @Override
+ public Iterator<VectorWrapper<?>> iterator() {
+ return container.iterator();
+ }
+
+ public void clear() {
+ container.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
index 428632f..da189eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
@@ -19,13 +19,21 @@ package org.apache.drill.exec.physical.impl.window;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import com.google.common.collect.Maps;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JVar;
import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.logical.data.NamedExpression;
import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
@@ -33,13 +41,11 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.memory.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.WindowPOP;
-import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.record.AbstractRecordBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
@@ -61,12 +67,51 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WindowFrameRecordBatch.class);
private final RecordBatch incoming;
- private List<RecordBatchData> batches;
+ private List<WindowDataBatch> batches;
private WindowFramer framer;
private boolean noMoreBatches;
private BatchSchema schema;
+ /**
+ * Describes supported window functions and if they output FLOAT8 or BIGINT
+ */
+ private enum WindowFunction {
+ ROW_NUMBER(false),
+ RANK(false),
+ DENSE_RANK(false),
+ PERCENT_RANK(true),
+ CUME_DIST(true);
+
+ private final boolean useDouble;
+
+ WindowFunction(boolean useDouble) {
+ this.useDouble = useDouble;
+ }
+
+ public TypeProtos.MajorType getMajorType() {
+ return useDouble ? Types.required(TypeProtos.MinorType.FLOAT8) : Types.required(TypeProtos.MinorType.BIGINT);
+ }
+
+ /**
+ * Extract the WindowFunction corresponding to the logical expression
+ * @param expr logical expression
+ * @return WindowFunction or null if the logical expression is not a window function
+ */
+ public static WindowFunction fromExpression(final LogicalExpression expr) {
+ if (!(expr instanceof FunctionCall)) {
+ return null;
+ }
+
+ final String name = ((FunctionCall) expr).getName();
+ try {
+ return WindowFunction.valueOf(name.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return null; // not a window function
+ }
+ }
+ }
+
public WindowFrameRecordBatch(WindowPOP popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
super(popConfig, context);
this.incoming = incoming;
@@ -90,7 +135,7 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
*
* <p><pre>
* when innerNext() is called:
- * call next(incoming), we receive and save b0 in a list of RecordDataBatch
+ * call next(incoming), we receive and save b0 in a list of WindowDataBatch
* we can't process b0 yet because we don't know if p1 has more rows upstream
* call next(incoming), we receive and save b1
* we can't process b0 yet for the same reason previously stated
@@ -107,7 +152,11 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
* when innerNext() is called:
* we return NONE
* </pre></p>
- *
+ * The previous scenario applies when we don't have an ORDER BY clause, otherwise a batch can be processed
+ * as soon as we reach the final peer row of the batch's last row (we find the end of the last frame of the batch).
+ * </p>
+ * Because we only support the default frame, we don't need to reset the aggregations until we reach the end of
+ * a partition. We can safely free a batch as soon as it has been processed.
*/
@Override
public IterOutcome innerNext() {
@@ -130,24 +179,21 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
case OUT_OF_MEMORY:
case NOT_YET:
case STOP:
+ cleanup();
return upstream;
case OK_NEW_SCHEMA:
- // when a partition of rows exceeds the current processed batch, it will be kept as "pending" and processed
- // when innerNext() is called again. If the schema changes, the framer is "rebuilt" and the pending information
- // will be lost which may lead to incorrect results.
-
- // only change in the case that the schema truly changes. Artificial schema changes are ignored.
+ // We don't support schema changes
if (!incoming.getSchema().equals(schema)) {
if (schema != null) {
- throw new UnsupportedOperationException("Sort doesn't currently support sorts with changing schemas.");
+ throw new UnsupportedOperationException("OVER clause doesn't currently support changing schemas.");
}
this.schema = incoming.getSchema();
}
case OK:
- batches.add(new RecordBatchData(incoming));
+ batches.add(new WindowDataBatch(incoming, context));
break;
default:
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException("Unsupported upstrean state " + upstream);
}
}
@@ -157,15 +203,12 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
return IterOutcome.NONE;
}
- // process a saved batch
+ // process first saved batch, then release it
try {
framer.doWork();
} catch (DrillException e) {
context.fail(e);
- if (framer != null) {
- framer.cleanup();
- framer = null;
- }
+ cleanup();
return IterOutcome.STOP;
}
@@ -201,52 +244,44 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
}
private WindowFramer createFramer(VectorAccessible batch) throws SchemaChangeException, IOException, ClassTransformationException {
- logger.trace("creating framer");
+ assert framer == null : "createFramer should only be called once";
- container.clear();
+ logger.trace("creating framer");
- if (framer != null) {
- framer.cleanup();
- framer = null;
- }
+ final List<LogicalExpression> aggExprs = Lists.newArrayList();
+ final Map<WindowFunction, TypedFieldId> winExprs = Maps.newHashMap();
+ final List<LogicalExpression> keyExprs = Lists.newArrayList();
+ final List<LogicalExpression> orderExprs = Lists.newArrayList();
+ final ErrorCollector collector = new ErrorCollectorImpl();
- ErrorCollector collector = new ErrorCollectorImpl();
+ container.clear();
- // setup code generation to copy all incoming vectors to the container
- // we can't just transfer them because after we pass the container downstream, some values will be needed when
- // processing the next batches
- int j = 0;
- LogicalExpression[] windowExprs = new LogicalExpression[batch.getSchema().getFieldCount()];
+ // all existing vectors will be transferred to the outgoing container in framer.doWork()
for (VectorWrapper wrapper : batch) {
- // read value from saved batch
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(
- new ValueVectorReadExpression(new TypedFieldId(wrapper.getField().getType(), wrapper.isHyper(), j)),
- batch, collector, context.getFunctionRegistry());
-
- ValueVector vv = container.addOrGet(wrapper.getField());
- vv.allocateNew();
-
- // write value into container
- TypedFieldId id = container.getValueVectorId(vv.getField().getPath());
- windowExprs[j] = new ValueVectorWriteExpression(id, expr, true);
- j++;
+ container.addOrGet(wrapper.getField());
}
// add aggregation vectors to the container, and materialize corresponding expressions
- LogicalExpression[] aggExprs = new LogicalExpression[popConfig.getAggregations().length];
- for (int i = 0; i < aggExprs.length; i++) {
- // evaluate expression over saved batch
- NamedExpression ne = popConfig.getAggregations()[i];
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), batch, collector, context.getFunctionRegistry());
-
- // add corresponding ValueVector to container
- final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
- ValueVector vv = container.addOrGet(outputField);
- vv.allocateNew();
-
- // write value into container
- TypedFieldId id = container.getValueVectorId(ne.getRef());
- aggExprs[i] = new ValueVectorWriteExpression(id, expr, true);
+ for (final NamedExpression ne : popConfig.getAggregations()) {
+ final WindowFunction wf = WindowFunction.fromExpression(ne.getExpr());
+
+ if (wf != null) {
+ // add corresponding ValueVector to container
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), wf.getMajorType());
+ ValueVector vv = container.addOrGet(outputField);
+ vv.allocateNew();
+ winExprs.put(wf, container.getValueVectorId(ne.getRef()));
+ } else {
+ // evaluate expression over saved batch
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), batch, collector, context.getFunctionRegistry());
+
+ // add corresponding ValueVector to container
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ ValueVector vv = container.addOrGet(outputField);
+ vv.allocateNew();
+ TypedFieldId id = container.getValueVectorId(ne.getRef());
+ aggExprs.add(new ValueVectorWriteExpression(id, expr, true));
+ }
}
if (container.isSchemaChanged()) {
@@ -254,17 +289,15 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
}
// materialize partition by expressions
- LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getWithins().length];
- for (int i = 0; i < keyExprs.length; i++) {
- NamedExpression ne = popConfig.getWithins()[i];
- keyExprs[i] = ExpressionTreeMaterializer.materialize(ne.getExpr(), batch, collector, context.getFunctionRegistry());
+ for (final NamedExpression ne : popConfig.getWithins()) {
+ keyExprs.add(
+ ExpressionTreeMaterializer.materialize(ne.getExpr(), batch, collector, context.getFunctionRegistry()));
}
// materialize order by expressions
- LogicalExpression[] orderExprs = new LogicalExpression[popConfig.getOrderings().length];
- for (int i = 0; i < orderExprs.length; i++) {
- Order.Ordering oe = popConfig.getOrderings()[i];
- orderExprs[i] = ExpressionTreeMaterializer.materialize(oe.getExpr(), batch, collector, context.getFunctionRegistry());
+ for (final Order.Ordering oe : popConfig.getOrderings()) {
+ orderExprs.add(
+ ExpressionTreeMaterializer.materialize(oe.getExpr(), batch, collector, context.getFunctionRegistry()));
}
if (collector.hasErrors()) {
@@ -272,14 +305,11 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
}
// generate framer code
-
final ClassGenerator<WindowFramer> cg = CodeGenerator.getRoot(WindowFramer.TEMPLATE_DEFINITION, context.getFunctionRegistry());
- // setup for isSamePartition()
- setupIsFunction(cg, keyExprs, isaB1, isaB2);
- // setup for isPeer()
- setupIsFunction(cg, orderExprs, isaP1, isaP2);
- setupAddRecords(cg, aggExprs);
- setupOutputWindowValues(cg, windowExprs);
+ setupIsFunction(cg, keyExprs, isaB1, isaB2); // setup for isSamePartition()
+ setupIsFunction(cg, orderExprs, isaP1, isaP2); // setup for isPeer()
+ setupOutputAggregatedValues(cg, aggExprs);
+ setupAddWindowValue(cg, winExprs);
cg.getBlock("resetValues")._return(JExpr.TRUE);
@@ -300,7 +330,8 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
/**
* setup comparison functions isSamePartition and isPeer
*/
- private void setupIsFunction(ClassGenerator<WindowFramer> cg, LogicalExpression[] exprs, MappingSet leftMapping, MappingSet rightMapping) {
+ private void setupIsFunction(final ClassGenerator<WindowFramer> cg, final List<LogicalExpression> exprs,
+ final MappingSet leftMapping, final MappingSet rightMapping) {
cg.setMappingSet(leftMapping);
for (LogicalExpression expr : exprs) {
cg.setMappingSet(leftMapping);
@@ -317,36 +348,51 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
cg.getEvalBlock()._return(JExpr.TRUE);
}
- private static final GeneratorMapping EVAL_INSIDE = GeneratorMapping.create("setupIncoming", "addRecord", null, null);
- private static final GeneratorMapping EVAL_OUTSIDE = GeneratorMapping.create("setupOutgoing", "outputRecordValues", "resetValues", "cleanup");
+ private static final GeneratorMapping EVAL_INSIDE = GeneratorMapping.create("setupRead", "aggregateRecord", null, null);
+ private static final GeneratorMapping EVAL_OUTSIDE = GeneratorMapping.create("setupWrite", "outputAggregatedValues", "resetValues", "cleanup");
private final MappingSet eval = new MappingSet("index", "outIndex", EVAL_INSIDE, EVAL_OUTSIDE, EVAL_INSIDE);
/**
- * setup for addRecords() and outputRecordValues()
+ * setup for aggregateRecord() and outputAggregatedValues()
*/
- private void setupAddRecords(ClassGenerator<WindowFramer> cg, LogicalExpression[] valueExprs) {
+ private void setupOutputAggregatedValues(ClassGenerator<WindowFramer> cg, List<LogicalExpression> valueExprs) {
cg.setMappingSet(eval);
for (LogicalExpression ex : valueExprs) {
cg.addExpr(ex);
}
}
- private final static GeneratorMapping OUTPUT_WINDOW_VALUES = GeneratorMapping.create("setupCopy", "outputWindowValues", null, null);
- private final MappingSet windowValues = new MappingSet("index", "index", OUTPUT_WINDOW_VALUES, OUTPUT_WINDOW_VALUES);
-
- private void setupOutputWindowValues(ClassGenerator<WindowFramer> cg, LogicalExpression[] valueExprs) {
- cg.setMappingSet(windowValues);
- for (LogicalExpression valueExpr : valueExprs) {
- cg.addExpr(valueExpr);
+ /**
+ * generate code to write "computed" window function values into their respective value vectors
+ */
+ private void setupAddWindowValue(final ClassGenerator<WindowFramer> cg, final Map<WindowFunction, TypedFieldId> functions) {
+ cg.setMappingSet(eval);
+ for (WindowFunction function : functions.keySet()) {
+ final JVar vv = cg.declareVectorValueSetupAndMember(cg.getMappingSet().getOutgoing(), functions.get(function));
+ final JExpression outIndex = cg.getMappingSet().getValueWriteIndex();
+ JInvocation setMethod = vv.invoke("getMutator").invoke("setSafe").arg(outIndex).arg(
+ JExpr.direct("partition." + function.name().toLowerCase()));
+ cg.getEvalBlock().add(setMethod);
}
}
- @Override
- public void close() {
+ private void cleanup() {
if (framer != null) {
framer.cleanup();
framer = null;
}
+
+ if (batches != null) {
+ for (final WindowDataBatch bd : batches) {
+ bd.clear();
+ }
+ batches = null;
+ }
+ }
+
+ @Override
+ public void close() {
+ cleanup();
super.close();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameTemplate.java
deleted file mode 100644
index 78bab54..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameTemplate.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.window;
-
-import org.apache.drill.common.exceptions.DrillException;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.record.VectorAccessible;
-import org.apache.drill.exec.record.VectorWrapper;
-
-import javax.inject.Named;
-import java.util.Iterator;
-import java.util.List;
-
-
-public abstract class WindowFrameTemplate implements WindowFramer {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WindowFrameTemplate.class);
-
- private VectorAccessible container;
- private List<RecordBatchData> batches;
- private int outputCount; // number of rows in currently/last processed batch
-
- /**
- * current partition being processed. Can span over multiple batches, so we may need to keep it between calls to doWork()
- */
- private Interval partition;
-
- private int currentBatch; // first unprocessed batch
-
- @Override
- public void setup(List<RecordBatchData> batches, VectorAccessible container) throws SchemaChangeException {
- this.container = container;
- this.batches = batches;
-
- outputCount = 0;
- partition = null;
- currentBatch = 0;
-
- setupOutgoing(container);
- }
-
- /**
- * processes all rows of current batch:
- * <ul>
- * <li>compute window aggregations</li>
- * <li>copy remaining vectors from current batch to container</li>
- * </ul>
- */
- @Override
- public void doWork() throws DrillException {
- logger.trace("WindowFramer.doWork() START, num batches {}, currentBatch {}", batches.size(), currentBatch);
-
- final VectorAccessible current = batches.get(currentBatch).getContainer();
-
- // we need to store the record count explicitly, in case we release current batch at the end of this call
- outputCount = current.getRecordCount();
-
- // allocate vectors
- for (VectorWrapper<?> w : container){
- w.getValueVector().allocateNew();
- }
-
- setupCopy(current, container);
-
- int currentRow = 0;
-
- while (currentRow < outputCount) {
- if (partition != null) {
- assert currentRow == 0 : "pending windows are only expected at the start of the batch";
-
- // we have a pending window we need to handle from a previous call to doWork()
- logger.trace("we have a pending partition {}", partition);
- } else {
- // compute the size of the new partition
- final int length = computePartitionSize(currentRow);
- partition = new Interval(currentRow, length, 0);
- }
-
- currentRow = processPartition(currentRow);
-
- if (partition == null) {
- freeBatches(currentRow == outputCount);
- }
- }
-
- if (partition != null) {
- logger.trace("we have a pending partition {}", partition);
- // current batch has been processed but it won't be released until this pending partition is processed
- currentBatch++;
- }
-
- for (VectorWrapper<?> v : container) {
- v.getValueVector().getMutator().setValueCount(outputCount);
- }
-
- logger.trace("WindowFramer.doWork() END");
- }
-
- /**
- * releases saved batches that are no longer needed: all rows of their partitions have been processed
- * @param freeCurrent do we free current batch too ?
- */
- private void freeBatches(boolean freeCurrent) {
- // how many batches can be released
- int numFree = currentBatch;
- if (freeCurrent) { // current batch can also be released
- numFree++;
- }
- if (numFree > 0) {
- logger.trace("freeing {} batches", numFree);
-
- // we are ready to free batches < currentBatch
- for (int i = 0; i < numFree; i++) {
- RecordBatchData bd = batches.remove(0);
- bd.getContainer().clear();
- }
-
- currentBatch = 0;
- }
- }
-
- /**
- * process all rows (computes and writes aggregation values) of current batch that are part of current partition.
- * @return index of next unprocessed row
- * @throws DrillException if it can't write into the container
- */
- private int processPartition(int currentRow) throws DrillException {
- logger.trace("process partition {}, currentBatch: {}, currentRow: {}", partition, currentBatch, currentRow);
-
- // compute how many rows remain unprocessed in the current partition
- int remaining = partition.length;
- for (int b = 0; b < currentBatch; b++) {
- remaining -= batches.get(b).getRecordCount();
- }
- remaining -= currentRow - partition.start;
-
- // when computing the frame for the current row, keep track of how many peer rows need to be processed
- // because all peer rows share the same frame, we only need to compute and aggregate the frame once
- for (int peers = 0; currentRow < outputCount && remaining > 0; currentRow++, remaining--) {
- if (peers == 0) {
- Interval frame = computeFrame(currentBatch, currentRow);
- resetValues();
- aggregate(frame);
- peers = frame.peers;
- } else {
- peers--;
- }
-
- outputRecordValues(currentRow);
- outputWindowValues(currentRow);
- }
-
- if (remaining == 0) {
- logger.trace("finished processing {}", partition);
- partition = null;
- }
-
- return currentRow;
- }
-
- /**
- * @return number of rows that are part of the partition starting at row start of first batch
- */
- private int computePartitionSize(int start) {
- logger.trace("compute partition size starting from {} on {} batches", start, batches.size());
-
- // current partition always starts from first batch
- final VectorAccessible first = batches.get(0).getContainer();
-
- int length = 0;
-
- // count all rows that are in the same partition of start
- outer:
- for (RecordBatchData batch : batches) {
- final VectorAccessible cont = batch.getContainer();
-
- // check first container from start row, and subsequent containers from first row
- for (int row = (cont == first ? start : 0); row < cont.getRecordCount(); row++) {
- if (isSamePartition(start, first, row, cont)) {
- length++;
- } else {
- break outer;
- }
- }
- }
-
- return length;
- }
-
- /**
- * find the limits of the window frame for a row
- * @param batchId batch where the current row is
- * @param row idx of row in the given batch
- * @return frame interval
- */
- private Interval computeFrame(int batchId, int row) {
-
- // using default frame for now RANGE BETWEEN UNBOUND PRECEDING AND CURRENT ROW
- // frame contains all rows from start of partition to last peer of row
-
- // count how many rows in the current partition precede the row
- int length = row;
- // include rows of all batches previous to batchId
- Iterator<RecordBatchData> iterator = batches.iterator();
- for (int b = 0; b < batchId; b++) {
- length += iterator.next().getRecordCount();
- }
- length -= partition.start;
-
- VectorAccessible batch = iterator.next().getContainer();
- VectorAccessible current = batch;
-
- // for every remaining row in the partition, count it if it's a peer row
- int peers = 0;
- for (int curRow = row; length < partition.length; length++, curRow++, peers++) {
- if (curRow == current.getRecordCount()) {
- current = iterator.next().getContainer();
- curRow = 0;
- }
-
- if (!isPeer(row, batch, curRow, current)) {
- break;
- }
- }
-
- // do not count row as a peer
- return new Interval(partition.start, length, peers-1);
- }
-
- private void aggregate(Interval frame) throws SchemaChangeException {
- logger.trace("aggregating {}", frame);
- assert frame.length > 0 : "processing empty frame!";
-
- // a single frame can include rows from multiple batches
- // start processing first batch and, if necessary, move to next batches
- Iterator<RecordBatchData> iterator = batches.iterator();
- VectorAccessible current = iterator.next().getContainer();
- setupIncoming(current);
-
- for (int i = 0, row = frame.start; i < frame.length; i++, row++) {
- if (row >= current.getRecordCount()) {
- // we reached the end of the current batch, move to the next one
- current = iterator.next().getContainer();
- setupIncoming(current);
- row = 0;
- }
-
- addRecord(row);
- }
- }
-
- @Override
- public boolean canDoWork() {
- // check if we can process a saved batch
- if (batches.size() > 1) {
- final VectorAccessible last = batches.get(batches.size()-1).getContainer();
-
- if (!isSamePartition(getCurrent().getRecordCount() - 1, getCurrent(), last.getRecordCount() - 1, last)) {
- logger.trace("partition changed, we are ready to process first saved batch");
- return true;
- } else {
- logger.trace("partition didn't change, fetch next batch");
- }
- } else {
- logger.trace("we don't have enough batches to proceed, fetch next batch");
- }
-
- return false;
- }
-
- @Override
- public VectorAccessible getCurrent() {
- return batches.get(currentBatch).getContainer();
- }
-
- @Override
- public int getOutputCount() {
- return outputCount;
- }
-
- @Override
- public void cleanup() {
- }
-
- /**
- * setup incoming container for addRecord()
- */
- public abstract void setupIncoming(@Named("incoming") VectorAccessible incoming) throws SchemaChangeException;
-
- /**
- * setup outgoing container for outputRecordValues
- */
- public abstract void setupOutgoing(@Named("outgoing") VectorAccessible outgoing) throws SchemaChangeException;
-
- /**
- * setup for outputWindowValues
- */
- public abstract void setupCopy(@Named("incoming") VectorAccessible incoming, @Named("outgoing") VectorAccessible outgoing) throws SchemaChangeException;
-
- /**
- * aggregates a row from the incoming container
- * @param index of row to aggregate
- */
- public abstract void addRecord(@Named("index") int index);
-
- /**
- * writes aggregated values to row of outgoing container
- * @param outIndex index of row
- */
- public abstract void outputRecordValues(@Named("outIndex") int outIndex);
-
- /**
- * copies all value vectors from incoming to container, for a specific row
- * @param index of row to be copied
- */
- public abstract void outputWindowValues(@Named("index") int index);
-
- /**
- * reset all window functions
- */
- public abstract boolean resetValues();
-
- /**
- * compares two rows from different batches (can be the same), if they have the same value for the partition by
- * expression
- * @param b1Index index of first row
- * @param b1 batch for first row
- * @param b2Index index of second row
- * @param b2 batch for second row
- * @return true if the rows are in the same partition
- */
- public abstract boolean isSamePartition(@Named("b1Index") int b1Index, @Named("b1") VectorAccessible b1, @Named("b2Index") int b2Index, @Named("b2") VectorAccessible b2);
-
- /**
- * compares two rows from different batches (can be the same), if they have the same value for the order by
- * expression
- * @param b1Index index of first row
- * @param b1 batch for first row
- * @param b2Index index of second row
- * @param b2 batch for second row
- * @return true if the rows are in the same partition
- */
- public abstract boolean isPeer(@Named("b1Index") int b1Index, @Named("b1") VectorAccessible b1, @Named("b2Index") int b2Index, @Named("b2") VectorAccessible b2);
-
- /**
- * Used internally to keep track of partitions and frames
- */
- private static class Interval {
- public final int start;
- public final int length;
- public final int peers; // we only need this for frames
-
- public Interval(int start, int length, int peers) {
- this.start = start;
- this.length = length;
- this.peers = peers;
- }
-
- @Override
- public String toString() {
- return String.format("{start: %d, length: %d, peers: %d}", start, length, peers);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFramer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFramer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFramer.java
index 23a2b53..69866af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFramer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFramer.java
@@ -20,21 +20,20 @@ package org.apache.drill.exec.physical.impl.window;
import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
-import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorContainer;
import java.util.List;
public interface WindowFramer {
- public static TemplateClassDefinition<WindowFramer> TEMPLATE_DEFINITION = new TemplateClassDefinition<>(WindowFramer.class, WindowFrameTemplate.class);
+ TemplateClassDefinition<WindowFramer> TEMPLATE_DEFINITION = new TemplateClassDefinition<>(WindowFramer.class, DefaultFrameTemplate.class);
- public abstract void setup(List<RecordBatchData> batches, VectorAccessible container) throws SchemaChangeException;
+ void setup(List<WindowDataBatch> batches, final VectorContainer container) throws SchemaChangeException;
/**
* process the inner batch and write the aggregated values in the container
* @throws DrillException
*/
- public abstract void doWork() throws DrillException;
+ void doWork() throws DrillException;
/**
* check if current batch can be processed:
@@ -44,16 +43,12 @@ public interface WindowFramer {
* </ol>
* @return true if current batch can be processed, false otherwise
*/
- public abstract boolean canDoWork();
+ boolean canDoWork();
+
/**
* @return number rows processed in last batch
*/
- public abstract int getOutputCount();
-
- public abstract void cleanup();
+ int getOutputCount();
- /**
- * @return saved batch that will be processed in doWork()
- */
- public abstract VectorAccessible getCurrent();
+ void cleanup();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
index 9b21ae3..6686fbe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
@@ -17,12 +17,14 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
+import com.typesafe.config.ConfigException;
import io.netty.buffer.DrillBuf;
import java.util.Queue;
import javax.inject.Named;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
@@ -36,7 +38,7 @@ import com.google.common.base.Stopwatch;
import com.google.common.collect.Queues;
public abstract class MSortTemplate implements MSorter, IndexedSortable{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MSortTemplate.class);
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MSortTemplate.class);
private SelectionVector4 vector4;
private SelectionVector4 aux;
@@ -68,7 +70,16 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
}
}
final DrillBuf drillBuf = allocator.buffer(4 * totalCount);
- aux = new SelectionVector4(drillBuf, totalCount, Character.MAX_VALUE);
+
+ // This is only useful for debugging: change the maximum size of batches exposed to downstream
+ // when we don't spill to disk
+ int MSORT_BATCH_MAXSIZE;
+ try {
+ MSORT_BATCH_MAXSIZE = context.getConfig().getInt(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE);
+ } catch(ConfigException.Missing e) {
+ MSORT_BATCH_MAXSIZE = Character.MAX_VALUE;
+ }
+ aux = new SelectionVector4(drillBuf, totalCount, MSORT_BATCH_MAXSIZE);
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index 330ec79..ff53052 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.record.selection.SelectionVector4;
public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements CloseableRecordBatch {
final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(this.getClass());
- protected final VectorContainer container; //= new VectorContainer();
+ protected final VectorContainer container;
protected final T popConfig;
protected final FragmentContext context;
protected final OperatorContext oContext;
@@ -56,8 +56,8 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
this.context = context;
this.popConfig = popConfig;
this.oContext = oContext;
- this.stats = oContext.getStats();
- this.container = new VectorContainer(this.oContext);
+ stats = oContext.getStats();
+ container = new VectorContainer(this.oContext);
if (buildSchema) {
state = BatchState.BUILD_SCHEMA;
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
index adbf653..5d8cd95 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
@@ -23,7 +23,6 @@ import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedDataTypeException;
import org.apache.drill.exec.work.foreman.UnsupportedFunctionException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
-import org.junit.Ignore;
import org.junit.Test;
public class TestDisabledFunctionality extends BaseTestQuery{
@@ -286,18 +285,6 @@ public class TestDisabledFunctionality extends BaseTestQuery{
}
}
- @Test(expected = UnsupportedFunctionException.class) // see DRILL-2441
- public void testDisabledWindowFunctions() throws Exception {
- try {
- test("SELECT employee_id,position_id, salary, avg(salary) " +
- "OVER (PARTITION BY position_id order by position_id) " +
- "FROM cp.`employee.json` " +
- "order by employee_id;");
- } catch(UserException ex) {
- throwAsUnsupportedException(ex);
- }
- }
-
@Test(expected = UnsupportedFunctionException.class) // see DRILL-2181
public void testFlattenWithinGroupBy() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
new file mode 100644
index 0000000..623c1e2
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.window;
+
+import org.apache.drill.common.util.TestTools;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class GenerateTestData {
+ private static final int SUB_MUL = 1;
+ private static final int BATCH_SIZE = 20;
+
+ private static class Partition {
+ Partition previous;
+ final int length;
+ final int[] subs;
+
+ public Partition(int length, int[] subs) {
+ this.length = length;
+ this.subs = subs;
+ }
+
+ /**
+ * @return total number of rows since first partition, this partition included
+ */
+ public int cumulLength() {
+ int prevLength = previous != null ? previous.cumulLength() : 0;
+ return length + prevLength;
+ }
+
+ public boolean isPartOf(int rowNumber) {
+ int prevLength = previous != null ? previous.cumulLength() : 0;
+ return rowNumber >= prevLength && rowNumber < cumulLength();
+ }
+
+ public int getSubIndex(final int sub) {
+ return Arrays.binarySearch(subs, sub);
+ }
+
+ public int getSubSize(int sub) {
+ if (sub != subs[subs.length-1]) {
+ return sub * SUB_MUL;
+ } else {
+ //last sub has enough rows to reach partition length
+ int size = length;
+ for (int i = 0; i < subs.length-1; i++) {
+ size -= subs[i] * SUB_MUL;
+ }
+ return size;
+ }
+ }
+
+ /**
+ * @return sub id of the sub that contains rowNumber
+ */
+ public int getSubId(int rowNumber) {
+ assert isPartOf(rowNumber) : "row "+rowNumber+" isn't part of this partition";
+
+ int prevLength = previous != null ? previous.cumulLength() : 0;
+ rowNumber -= prevLength; // row num from start of this partition
+
+ for (int s : subs) {
+ if (rowNumber < subRunningCount(s)) {
+ return s;
+ }
+ }
+
+ throw new RuntimeException("should never happen!");
+ }
+
+ /**
+ * @return running count of rows from first row of the partition to current sub, this sub included
+ */
+ public int subRunningCount(int sub) {
+ int count = 0;
+ for (int s : subs) {
+ count += getSubSize(s);
+ if (s == sub) {
+ break;
+ }
+ }
+ return count;
+ }
+
+ /**
+ * @return running sum of salaries from first row of the partition to current sub, this sub included
+ */
+ public int subRunningSum(int sub) {
+ int sum = 0;
+ for (int s : subs) {
+ sum += (s+10) * getSubSize(s);
+ if (s == sub) {
+ break;
+ }
+ }
+ return sum;
+ }
+
+ /**
+ * @return sum of salaries for all rows of the partition
+ */
+ public int totalSalary() {
+ return subRunningSum(subs[subs.length-1]);
+ }
+
+ }
+
+ private static Partition[] dataB1P1() {
+ // partition rows 20, subs [1, 2, 3, 4, 5, 6]
+ return new Partition[] {
+ new Partition(20, new int[]{1, 2, 3, 4, 5, 6})
+ };
+ }
+
+ private static Partition[] dataB1P2() {
+ // partition rows 10, subs [1, 2, 3, 4]
+ // partition rows 10, subs [4, 5, 6]
+ return new Partition[] {
+ new Partition(10, new int[]{1, 2, 3, 4}),
+ new Partition(10, new int[]{4, 5, 6}),
+ };
+ }
+
+ private static Partition[] dataB2P2() {
+ // partition rows 20, subs [3, 5, 9]
+ // partition rows 20, subs [9, 10]
+ return new Partition[] {
+ new Partition(20, new int[]{3, 5, 9}),
+ new Partition(20, new int[]{9, 10}),
+ };
+ }
+
+ private static Partition[] dataB2P4() {
+ // partition rows 5, subs [1, 2, 3]
+ // partition rows 10, subs [3, 4, 5]
+ // partition rows 15, subs [5, 6, 7]
+ // partition rows 10, subs [7, 8]
+ return new Partition[] {
+ new Partition(5, new int[]{1, 2, 3}),
+ new Partition(10, new int[]{3, 4, 5}),
+ new Partition(15, new int[]{5, 6, 7}),
+ new Partition(10, new int[]{7, 8}),
+ };
+ }
+
+ private static Partition[] dataB3P2() {
+ // partition rows 5, subs [1, 2, 3]
+ // partition rows 55, subs [4, 5, 7, 8, 9, 10, 11, 12]
+ return new Partition[] {
+ new Partition(5, new int[]{1, 2, 3}),
+ new Partition(55, new int[]{4, 5, 7, 8, 9, 10, 11, 12}),
+ };
+ }
+
+ private static Partition[] dataB4P4() {
+ // partition rows 10, subs [1, 2, 3]
+ // partition rows 30, subs [3, 4, 5, 6, 7, 8]
+ // partition rows 20, subs [8, 9, 10]
+ // partition rows 20, subs [10, 11]
+ return new Partition[] {
+ new Partition(10, new int[]{1, 2, 3}),
+ new Partition(30, new int[]{3, 4, 5, 6, 7, 8}),
+ new Partition(20, new int[]{8, 9, 10}),
+ new Partition(20, new int[]{10, 11}),
+ };
+ }
+
+ private static void generateData(final String tableName, final Partition[] partitions) throws FileNotFoundException {
+ final String WORKING_PATH = TestTools.getWorkingPath();
+ final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+ //TODO command line arguments contain file name
+ final String path = TEST_RES_PATH+"/window/" + tableName;
+
+ final File pathFolder = new File(path);
+ if (!pathFolder.exists()) {
+ if (!pathFolder.mkdirs()) {
+ System.err.printf("Couldn't create folder %s, exiting%n", path);
+ }
+ }
+
+ // expected results for query without order by clause
+ final PrintStream resultStream = new PrintStream(path + ".tsv");
+ // expected results for query with order by clause
+ final PrintStream resultOrderStream = new PrintStream(path + ".subs.tsv");
+
+ // data file(s)
+ int fileId = 0;
+ PrintStream dataStream = new PrintStream(path + "/" + fileId + ".data.json");
+
+ for (Partition p : partitions) {
+ dataStream.printf("// partition rows %d, subs %s%n", p.length, Arrays.toString(p.subs));
+ }
+
+ // set previous partitions
+ for (int i = 1; i < partitions.length; i++) {
+ partitions[i].previous = partitions[i - 1];
+ }
+
+ // total number of rows
+ int total = partitions[partitions.length - 1].cumulLength();
+
+ // create data rows in randome order
+ List<Integer> emp_ids = new ArrayList<>(total);
+ for (int i = 0; i < total; i++) {
+ emp_ids.add(i);
+ }
+ Collections.shuffle(emp_ids);
+
+ int emp_idx = 0;
+ for (int id : emp_ids) {
+ int p = 0;
+ while (!partitions[p].isPartOf(id)) { // emp x is @ row x-1
+ p++;
+ }
+
+ int sub = partitions[p].getSubId(id);
+ int salary = 10 + sub;
+
+ dataStream.printf("{ \"employee_id\":%d, \"position_id\":%d, \"sub\":%d, \"salary\":%d }%n", id, p + 1, sub, salary);
+ emp_idx++;
+ if ((emp_idx % BATCH_SIZE)==0 && emp_idx < total) {
+ System.out.printf("total: %d, emp_idx: %d, fileID: %d%n", total, emp_idx, fileId);
+ dataStream.close();
+ fileId++;
+ dataStream = new PrintStream(path + "/" + fileId + ".data.json");
+ }
+ }
+
+ dataStream.close();
+
+ for (int p = 0, idx = 0; p < partitions.length; p++) {
+ for (int i = 0; i < partitions[p].length; i++, idx++) {
+ final Partition partition = partitions[p]; //TODO change for p loop to for over partitions
+
+ final int sub = partition.getSubId(idx);
+ final int rowNumber = i + 1;
+ final int rank = 1 + partition.subRunningCount(sub) - partition.getSubSize(sub);
+ final int denseRank = partition.getSubIndex(sub) + 1;
+ final double cumeDist = (double) partition.subRunningCount(sub) / partition.length;
+ final double percentRank = partition.length == 1 ? 0 : (double)(rank - 1)/(partition.length - 1);
+
+ // each line has: count(*) sum(salary) row_number() rank() dense_rank() cume_dist() percent_rank()
+ resultOrderStream.printf("%d\t%d\t%d\t%d\t%d\t%s\t%s%n",
+ partition.subRunningCount(sub), partition.subRunningSum(sub),
+ rowNumber, rank, denseRank, Double.toString(cumeDist), Double.toString(percentRank));
+
+ // each line has: count(*) sum(salary)
+ resultStream.printf("%d\t%d%n", partition.length, partition.totalSalary());
+ }
+ }
+
+ resultStream.close();
+ resultOrderStream.close();
+ }
+
+ public static void main(String[] args) throws FileNotFoundException {
+ generateData("b1.p1", dataB1P1());
+ generateData("b1.p2", dataB1P2());
+ generateData("b2.p2", dataB2P2());
+ generateData("b2.p4", dataB2P4());
+ generateData("b3.p2", dataB3P2());
+ generateData("b4.p4", dataB4P4());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
index 2b8bd64..15fefa5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
@@ -18,19 +18,60 @@
package org.apache.drill.exec.physical.impl.window;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.DrillTestWrapper;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
+import org.junit.BeforeClass;
import org.junit.Test;
+import java.util.Properties;
+
public class TestWindowFrame extends BaseTestQuery {
- private void runTest(String data, String results, String window) throws Exception {
- testNoResult("alter session set `%s`= true", ExecConstants.ENABLE_WINDOW_FUNCTIONS);
- testBuilder()
- .sqlQuery("select count(*) over pos_win `count`, sum(salary) over pos_win `sum` from cp.`window/%s.json` window pos_win as (%s)", data, window)
+ private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources";
+ private static final String QUERY_NO_ORDERBY =
+ "select count(*) over pos_win `count`, sum(salary) over pos_win `sum` from dfs_test.`%s/window/%s` window pos_win as (partition by position_id)";
+ private static final String QUERY_ORDERBY =
+ "select count(*) over pos_win `count`, sum(salary) over pos_win `sum`, row_number() over pos_win `row_number`, rank() over pos_win `rank`, dense_rank() over pos_win `dense_rank`, cume_dist() over pos_win `cume_dist`, percent_rank() over pos_win `percent_rank` from dfs_test.`%s/window/%s` window pos_win as (partition by position_id order by sub)";
+
+ @BeforeClass
+ public static void setupMSortBatchSize() {
+ // make sure memory sorter outputs 20 rows per batch
+ final Properties props = cloneDefaultTestConfigProperties();
+ props.put(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE, Integer.toString(20));
+
+ updateTestCluster(1, DrillConfig.create(props));
+ }
+
+ private DrillTestWrapper buildWindowQuery(final String tableName) throws Exception {
+ return testBuilder()
+ .sqlQuery(String.format(QUERY_NO_ORDERBY, TEST_RES_PATH, tableName))
.ordered()
- .csvBaselineFile("window/" + results + ".tsv")
+ .csvBaselineFile("window/" + tableName + ".tsv")
.baselineColumns("count", "sum")
- .build().run();
+ .build();
+ }
+
+ private DrillTestWrapper buildWindowWithOrderByQuery(final String tableName) throws Exception {
+ return testBuilder()
+ .sqlQuery(String.format(QUERY_ORDERBY, TEST_RES_PATH, tableName))
+ .ordered()
+ .csvBaselineFile("window/" + tableName + ".subs.tsv")
+ .baselineColumns("count", "sum", "row_number", "rank", "dense_rank", "cume_dist", "percent_rank")
+ .build();
+ }
+
+ private void runTest(final String tableName, final boolean withOrderBy) throws Exception {
+ runSQL(String.format("alter session set `%s`= true", ExecConstants.ENABLE_WINDOW_FUNCTIONS));
+
+ try {
+ DrillTestWrapper testWrapper = withOrderBy ?
+ buildWindowWithOrderByQuery(tableName) : buildWindowQuery(tableName);
+ testWrapper.run();
+ } finally {
+ runSQL(String.format("alter session set `%s`= false", ExecConstants.ENABLE_WINDOW_FUNCTIONS));
+ }
}
/**
@@ -38,7 +79,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB1P1() throws Exception {
- runTest("b1.p1.data", "b1.p1", "partition by position_id order by position_id");
+ runTest("b1.p1", false);
}
/**
@@ -46,7 +87,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB1P1OrderBy() throws Exception {
- runTest("b1.p1.data", "b1.p1.subs", "partition by position_id order by sub");
+ runTest("b1.p1", true);
}
/**
@@ -54,7 +95,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB1P2() throws Exception {
- runTest("b1.p2.data", "b1.p2", "partition by position_id order by position_id");
+ runTest("b1.p2", false);
}
/**
@@ -63,7 +104,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB1P2OrderBy() throws Exception {
- runTest("b1.p2.data", "b1.p2.subs", "partition by position_id order by sub");
+ runTest("b1.p2", true);
}
/**
@@ -71,12 +112,12 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB2P2() throws Exception {
- runTest("b2.p2.data", "b2.p2", "partition by position_id order by position_id");
+ runTest("b2.p2", false);
}
@Test
public void testB2P2OrderBy() throws Exception {
- runTest("b2.p2.data", "b2.p2.subs", "partition by position_id order by sub");
+ runTest("b2.p2", true);
}
/**
@@ -84,7 +125,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB2P4() throws Exception {
- runTest("b2.p4.data", "b2.p4", "partition by position_id order by position_id");
+ runTest("b2.p4", false);
}
/**
@@ -93,7 +134,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB2P4OrderBy() throws Exception {
- runTest("b2.p4.data", "b2.p4.subs", "partition by position_id order by sub");
+ runTest("b2.p4", true);
}
/**
@@ -101,7 +142,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB3P2() throws Exception {
- runTest("b3.p2.data", "b3.p2", "partition by position_id order by position_id");
+ runTest("b3.p2", false);
}
/**
@@ -110,7 +151,7 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testB3P2OrderBy() throws Exception {
- runTest("b3.p2.data", "b3.p2.subs", "partition by position_id order by sub");
+ runTest("b3.p2", true);
}
/**
@@ -119,7 +160,23 @@ public class TestWindowFrame extends BaseTestQuery {
*/
@Test
public void testb4P4() throws Exception {
- runTest("b4.p4.data", "b4.p4", "partition by position_id order by position_id");
+ runTest("b4.p4", false);
+ }
+
+ @Test
+ public void testb4P4OrderBy() throws Exception {
+ runTest("b4.p4", true);
}
+ @Test // DRILL-3218
+ public void testMaxVarChar() throws Exception {
+ runSQL(String.format("alter session set `%s`= true", ExecConstants.ENABLE_WINDOW_FUNCTIONS));
+
+ try {
+ test("select max(cast(columns[2] as char(2))) over(partition by cast(columns[2] as char(2)) order by cast(columns[0] as int)) from dfs_test.`%s/window/allData.csv`", TEST_RES_PATH);
+ } finally {
+ runSQL(String.format("alter session set `%s`= false", ExecConstants.ENABLE_WINDOW_FUNCTIONS));
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/allData.csv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/allData.csv b/exec/java-exec/src/test/resources/window/allData.csv
new file mode 100644
index 0000000..29ab38f
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/allData.csv
@@ -0,0 +1,5 @@
+-337516559,39342852852629160,VT,AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB,2014-06-02 00:28:02.418,1952-08-14,false,729363085.95,8:16:8.58
+406158122,81588677006971200,IN,AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB,2014-06-02 00:28:02.418,2001-03-08,false,1292460500.48,9:11:49.17
+1221407024,30009558124347168,VT,DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF,2014-06-02 00:28:02.419,2000-10-18,true,395110006.277,18:44:25.43
+-1609141704,47841997008600128,ND,GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI,2014-06-02 00:28:02.420,1991-05-13,true,1293582041.37,20:52:8.56
+-1032159521,38891661529640288,SD,HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ,2014-06-02 00:28:02.420,1965-02-21,false,983657842.924,19:46:10.42
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p1.subs.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p1.subs.tsv b/exec/java-exec/src/test/resources/window/b1.p1.subs.tsv
new file mode 100644
index 0000000..8368d4a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p1.subs.tsv
@@ -0,0 +1,20 @@
+1 11 1 1 1 0.05 0.0
+3 35 2 2 2 0.15 0.05263157894736842
+3 35 3 2 2 0.15 0.05263157894736842
+6 74 4 4 3 0.3 0.15789473684210525
+6 74 5 4 3 0.3 0.15789473684210525
+6 74 6 4 3 0.3 0.15789473684210525
+10 130 7 7 4 0.5 0.3157894736842105
+10 130 8 7 4 0.5 0.3157894736842105
+10 130 9 7 4 0.5 0.3157894736842105
+10 130 10 7 4 0.5 0.3157894736842105
+15 205 11 11 5 0.75 0.5263157894736842
+15 205 12 11 5 0.75 0.5263157894736842
+15 205 13 11 5 0.75 0.5263157894736842
+15 205 14 11 5 0.75 0.5263157894736842
+15 205 15 11 5 0.75 0.5263157894736842
+20 285 16 16 6 1.0 0.7894736842105263
+20 285 17 16 6 1.0 0.7894736842105263
+20 285 18 16 6 1.0 0.7894736842105263
+20 285 19 16 6 1.0 0.7894736842105263
+20 285 20 16 6 1.0 0.7894736842105263
http://git-wip-us.apache.org/repos/asf/drill/blob/3bccec91/exec/java-exec/src/test/resources/window/b1.p1.tsv
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/b1.p1.tsv b/exec/java-exec/src/test/resources/window/b1.p1.tsv
new file mode 100644
index 0000000..32f6ab7
--- /dev/null
+++ b/exec/java-exec/src/test/resources/window/b1.p1.tsv
@@ -0,0 +1,20 @@
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285
+20 285