You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by il...@apache.org on 2015/12/05 01:50:19 UTC
[6/8] incubator-asterixdb-hyracks git commit: Checkpointing: more or
less stable version
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProvider.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProvider.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProvider.java
new file mode 100644
index 0000000..c87c756
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProvider.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.common.api;
+
+public interface IOrdinalPrimitiveValueProvider {
+ public long getOrdinalValue(byte[] bytes, int offset);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProviderFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProviderFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProviderFactory.java
new file mode 100644
index 0000000..b479ba7
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IOrdinalPrimitiveValueProviderFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.common.api;
+
+import java.io.Serializable;
+
+public interface IOrdinalPrimitiveValueProviderFactory extends Serializable {
+ public IOrdinalPrimitiveValueProvider createOrdinalPrimitiveValueProvider();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProvider.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProvider.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProvider.java
deleted file mode 100644
index e5bd178..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProvider.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.api;
-
-public interface IPrimitiveIntegerValueProvider {
- public long getValue(byte[] bytes, int offset);
-
- public long minDomainValue(byte[] bytes, int offset);
-
- public long maxDomainValue(byte[] bytes, int offset);
-
- public int maxLevel(byte[] bytes, int offset);
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProviderFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProviderFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProviderFactory.java
deleted file mode 100644
index bbde1eb..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IPrimitiveIntegerValueProviderFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.api;
-
-import java.io.Serializable;
-
-public interface IPrimitiveIntegerValueProviderFactory extends Serializable {
- public IPrimitiveIntegerValueProvider createPrimitiveIntegerValueProvider();
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/data/PointableIntegerPrimitiveValueProviderFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/data/PointableIntegerPrimitiveValueProviderFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/data/PointableIntegerPrimitiveValueProviderFactory.java
deleted file mode 100644
index 8b211f8..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/data/PointableIntegerPrimitiveValueProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.common.data;
-
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.data.std.api.INumeric;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.api.IPointableFactory;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-
-public class PointableIntegerPrimitiveValueProviderFactory implements IPrimitiveValueProviderFactory {
- private static final long serialVersionUID = 1L;
-
- private final IPointableFactory pf;
- private final IPointable p;
- private final ITypeTraits traits;
-
- public PointableIntegerPrimitiveValueProviderFactory(IPointableFactory pf) {
- this.pf = pf;
- p = pf.createPointable();
- traits = pf.getTypeTraits();
- }
-
- @Override
- public IPrimitiveValueProvider createPrimitiveValueProvider() {
- final int length = traits.getFixedLength();
- return new IPrimitiveValueProvider() {
- @Override
- public double getValue(byte[] bytes, int offset) {
- p.set(bytes, offset, length);
- return ((INumeric) p).doubleValue();
- }
- };
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/StatisticsFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/StatisticsFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/StatisticsFactory.java
deleted file mode 100644
index 2e3e127..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/StatisticsFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
-import org.apache.hyracks.storage.am.common.statistics.wavelet.WaveletSynopsis;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-
-public class StatisticsFactory {
- private final IBufferCache bufferCache;
- private final IFileMapProvider fileMapProvider;
- private final int[] statisticsKeyFields;
- private final IPrimitiveIntegerValueProviderFactory valueProviderFactory;
-
- public StatisticsFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, int[] statisticsKeyFields,
- IPrimitiveIntegerValueProviderFactory valueProviderFactory) {
- this.bufferCache = bufferCache;
- this.fileMapProvider = fileMapProvider;
- this.statisticsKeyFields = statisticsKeyFields;
- this.valueProviderFactory = valueProviderFactory;
- }
-
- public WaveletSynopsis createWaveletStatistics(FileReference file) throws HyracksDataException {
- return new WaveletSynopsis(bufferCache, fileMapProvider, file, statisticsKeyFields, 10, valueProviderFactory);
- }
-
- public int[] getStatisticsFields() {
- return statisticsKeyFields;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/Synopsis.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/Synopsis.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/Synopsis.java
deleted file mode 100644
index 7b14822..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/Synopsis.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.impls.AbstractFileManager;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-
-public abstract class Synopsis<K, V> extends AbstractFileManager {
-
- public Synopsis(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file) {
- super(bufferCache, fileMapProvider, file);
- }
-
- public abstract void addElement(K key, V value);
-
- public abstract IIndexBulkLoader createBuilder() throws HyracksDataException;
-
- public abstract int getNumPages() throws HyracksDataException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/SynopsisType.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/SynopsisType.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/SynopsisType.java
deleted file mode 100644
index 6fda029..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/SynopsisType.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics;
-
-public enum SynopsisType {
- None,
- Wavelet,
- Sketch
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/GroupCountSketch.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/GroupCountSketch.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/GroupCountSketch.java
deleted file mode 100644
index c0d9d5c..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/GroupCountSketch.java
+++ /dev/null
@@ -1,90 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.sketch;
-
-public class GroupCountSketch extends Sketch {
-
- private final int levels;
- private final int depth;
- private final int buckets;
- private final int subbuckets;
- private final int fanoutLog;
- private final double counters[][][][];
- private final long[][] hashSeeds;
-
- public GroupCountSketch(int levels, int depth, int width, int fanoutLog/*double prob, double accuracy*/) {
- this.levels = levels;
- this.depth = depth;
- this.buckets = width;
- this.fanoutLog = fanoutLog;
- this.subbuckets = width * width;
-
- counters = new double[this.levels][this.depth][this.buckets][this.subbuckets];
- hashSeeds = new long[this.depth][8];
- initSeeds(this.depth, 8, hashSeeds);
- }
-
- public void update(long item, double diff) {
- int i, j, h, f, mult;
- long group;
-
- for (i = 0; i < depth; i++) {
- mult = HashGenerator.fourwise(this.hashSeeds[i][4], this.hashSeeds[i][5], this.hashSeeds[i][6],
- this.hashSeeds[i][7], item);
-
- f = HashGenerator.hash31(this.hashSeeds[i][2], this.hashSeeds[i][3], item);
- f = f % (this.subbuckets);
-
- for (j = 0, group = item; j < levels; j++, group >>= fanoutLog) {
-
- h = HashGenerator.hash31(this.hashSeeds[i][0], this.hashSeeds[i][1], group);
- h = h % (this.buckets);
-
- if ((mult & 1) == 1)
- this.counters[j][i][h][f] += diff;
- else
- this.counters[j][i][h][f] -= diff;
- }
- }
- }
-
- public double count(int group, int level) {
- int h, f, mult;
- double[] estimates = new double[depth];
-
- for (int i = 0; i < depth; i++) {
- h = HashGenerator.hash31(this.hashSeeds[i][0], this.hashSeeds[i][1], group);
- h = h % (this.buckets);
-
- f = HashGenerator.hash31(this.hashSeeds[i][2], this.hashSeeds[i][3], group);
- f = f % (this.subbuckets);
-
- mult = HashGenerator.fourwise(this.hashSeeds[i][4], this.hashSeeds[i][5], this.hashSeeds[i][6],
- this.hashSeeds[i][7], group);
- if ((mult & 1) == 1)
- estimates[i] += this.counters[level][i][h][f];
- else
- estimates[i] -= this.counters[level][i][h][f];
- }
-
- return getMedian(estimates, depth);
- }
-
- public double energyEst(int group, int level) {
- // estimate the F2 moment of the vector (sum of squares)
-
- int i, j;
- double z;
-
- double estimates[] = new double[depth];
- for (i = 0; i < depth; i++) {
- int h = HashGenerator.hash31(this.hashSeeds[i][0], this.hashSeeds[i][1], group);
- h = h % (this.buckets);
- z = 0;
- for (j = 0; j < this.subbuckets; j++) {
- z += Math.pow(this.counters[level][i][h][j], 2.0);
- }
- estimates[i] = z;
- }
-
- return getMedian(estimates, depth);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/HashGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/HashGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/HashGenerator.java
deleted file mode 100644
index 3ace6a0..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/HashGenerator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.sketch;
-
-public class HashGenerator {
-
- private static int MOD = 2147483647;
- private static int HL = 31;
-
- public static int hash31(long a, long b, long x) {
-
- long result;
-
- // return a hash of x using a and b mod (2^31 - 1)
- // may need to do another mod afterwards, or drop high bits
- // depending on d, number of bad guys
- // 2^31 - 1 = 2147483647
-
- // result = ((long long) a)*((long long) x)+((long long) b);
- result = (a * x) + b;
- result = ((result >> HL) + result) & MOD;
-
- return (int) result;
- }
-
- public static int fourwise(long a, long b, long c, long d, long x) {
- int result;
-
- // returns values that are 4-wise independent by repeated calls
- // to the pairwise independent routine.
-
- result = hash31(hash31(hash31(x, a, b), x, c), x, d);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/QuickSelect.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/QuickSelect.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/QuickSelect.java
deleted file mode 100644
index 5a05915..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/QuickSelect.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.sketch;
-
-import java.util.Random;
-
-public class QuickSelect {
-
- private static Random rand = new Random();
-
- private static int partition(double[] arr, int left, int right, int pivot) {
- double pivotVal = arr[pivot];
- swap(arr, pivot, right);
- int storeIndex = left;
- for (int i = left; i < right; i++) {
- if (arr[i] < pivotVal) {
- swap(arr, i, storeIndex);
- storeIndex++;
- }
- }
- swap(arr, right, storeIndex);
- return storeIndex;
- }
-
- public static double select(double[] arr, int n) {
- int left = 0;
- int right = arr.length - 1;
- while (right >= left) {
- int pivotIndex = partition(arr, left, right, rand.nextInt(right - left + 1) + left);
- if (pivotIndex == n) {
- return arr[pivotIndex];
- } else if (pivotIndex < n) {
- left = pivotIndex + 1;
- } else {
- right = pivotIndex - 1;
- }
- }
- return 0;
- }
-
- private static void swap(double[] arr, int i1, int i2) {
- if (i1 != i2) {
- double temp = arr[i1];
- arr[i1] = arr[i2];
- arr[i2] = temp;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/Sketch.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/Sketch.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/Sketch.java
deleted file mode 100644
index aff50ac..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/Sketch.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.sketch;
-
-import java.util.Random;
-
-public abstract class Sketch {
-
- protected void initSeeds(int k, int m, long[][] hashSeeds) {
- Random prng = new Random();
-
- int j, i;
- for (i = 0; i < k; i++) {
- for (j = 0; j < m; j++) {
- hashSeeds[i][j] = Math.abs(prng.nextLong()); //(int) prng.genInt();
- // initialise the hash functions
- // prng_int() should return a random integer
- // uniformly distributed in the range 0..2^31
- }
- }
- }
-
- protected static double getMedian(double[] data, int length) {
- return QuickSelect.select(data, data.length / 2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/SketchSynopsis.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/SketchSynopsis.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/SketchSynopsis.java
deleted file mode 100644
index 586943a..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/sketch/SketchSynopsis.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.sketch;
-
-//public class SketchSynopsis extends Synopsis implements IIndexBulkLoader {
-//
-// private final int levelNum;
-// private final int fanoutLog;
-// private final double epsilon;
-// private final GroupCountSketch gcSketch;
-// private final IBufferCache bufferCache;
-// private final IFileMapProvider fileMapProvider;
-// private final FileReference file;
-// private final int[] keyFields;
-// private final int fileId = -1;
-// private final boolean isActivated = false;
-//
-// public SketchSynopsis(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file,
-// int[] keyFields, int domainSize, double delta, double epsilon, int fanOut) {
-// super(bufferCache, fileMapProvider, file);
-// this.keyFields = keyFields;
-// this.fanoutLog = (int) (Math.log(fanOut) / Math.log(2.0));
-// this.levelNum = domainSize / fanoutLog;
-// this.epsilon = epsilon;
-// final int depth = (int) Math.ceil(Math.log(1 / delta));
-// final int width = (int) Math.ceil(1 / epsilon);
-// gcSketch = new GroupCountSketch(this.levelNum + 1, depth, width, fanoutLog);
-// }
-//
-// public void update(long item, double diff) {
-// //translate position to coefficient
-// item += 1 << (levelNum * fanoutLog);
-// //transform update into wavelet domain
-// long div = 1;
-// for (int i = 0; i < levelNum; i++) {
-// // Long coeffIdx = (long) ((1 << ((levelNum - i) * fanoutLog)) + item);
-// item >>= (fanoutLog - 1);
-// int sign = (item & 1) == 0 ? 1 : -1;
-// item >>= 1;
-// double normCoeff = WaveletCoefficient.getNormalizationCoefficient(levelNum * fanoutLog,
-// (i + 1) * fanoutLog);
-// div = (1 << ((i + 1) * fanoutLog));
-//
-// gcSketch.update(item, diff * sign / (normCoeff * div));
-// }
-// gcSketch.update(0, diff / div);
-// }
-//
-// @Override
-// public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
-// // TODO Auto-generated method stub
-//
-// }
-//
-// @Override
-// public void end() throws IndexException, HyracksDataException {
-// // TODO Auto-generated method stub
-//
-// }
-//
-// @Override
-// public void create() {
-// // TODO Auto-generated method stub
-//
-// }
-//
-// @Override
-// public void activate() {
-// // TODO Auto-generated method stub
-//
-// }
-//
-// @Override
-// public IIndexBulkLoader createBuilder(long numElements) {
-// // TODO Auto-generated method stub
-// return null;
-// }
-//
-// @Override
-// public void addElement(Object key, Object value) {
-// // TODO Auto-generated method stub
-//
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletCoefficient.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletCoefficient.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletCoefficient.java
deleted file mode 100644
index 5d9615b..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletCoefficient.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.wavelet;
-
-import java.util.Map.Entry;
-import java.util.Objects;
-
-import org.apache.hyracks.data.std.api.INumeric;
-
-public class WaveletCoefficient<K/*extends IMath<K>*/, V extends Comparable<V>/* extends IMath<V>*/>
- implements Entry<K, V>, Comparable<WaveletCoefficient<K, V>> {
-
- public V value;
- public int level;
- public K index;
-
- public WaveletCoefficient() {
- }
-
- public WaveletCoefficient(V value, int level, K index) {
- this.value = value;
- this.level = level;
- this.index = index;
- }
-
- @Override
- public K getKey() {
- return index;
- }
-
- @Override
- public V getValue() {
- return value;
- }
-
- @Override
- public V setValue(V value) {
- this.value = value;
- return this.value;
- }
-
- public void setLevel(int level) {
- this.level = level;
- }
-
- public void setIndex(K index) {
- this.index = index;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof WaveletCoefficient))
- return false;
- @SuppressWarnings("unchecked")
- WaveletCoefficient<K, V> triple = (WaveletCoefficient<K, V>) o;
- return triple.value.equals(value) && triple.level == level && triple.index.equals(index);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(value, level, index);
- }
-
- public static Double getNormalizationCoefficient(int maxLevel, int level) {
- return (1 << ((maxLevel - level) / 2)) * ((((maxLevel - level) % 2) == 0) ? 1 : Math.sqrt(2));
- }
-
- public static <K extends INumeric> int getLevel(K coeffPointable, int maxLevel) {
- long coeffIdx = coeffPointable.longValue();
- if (coeffIdx == 0)
- return maxLevel;
- int level = -1;
- while (coeffIdx > 0) {
- coeffIdx = coeffIdx >> 1;
- level++;
- }
- return maxLevel - level;
- }
-
- @Override
- // default comparator based on coefficient value
- public int compareTo(WaveletCoefficient<K, V> o) {
- return value.compareTo(o.getValue());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletSynopsis.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletSynopsis.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletSynopsis.java
deleted file mode 100644
index cae952d..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/statistics/wavelet/WaveletSynopsis.java
+++ /dev/null
@@ -1,263 +0,0 @@
-package org.apache.hyracks.storage.am.common.statistics.wavelet;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.PriorityQueue;
-import java.util.Stack;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProvider;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.statistics.Synopsis;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-
-public class WaveletSynopsis/*<K extends IMath<K>, V extends IMath<V>>*/ extends Synopsis<Integer, Double> {
-
- private final static int METADATA_PAGE_ID = 0;
- private final static int NUM_PAGES_OFFSET = 0;
- private final static int NUM_ELEMENTS_OFFSET = NUM_PAGES_OFFSET + 4;
-
- private final int[] waveletFields;
- private final IPrimitiveIntegerValueProvider waveletFieldValueProvider;
- private final PriorityQueue<WaveletCoefficient<Integer, Double>> coefficients;
- private final long threshold;
-
- private final int numPages;
-
- public WaveletSynopsis(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file,
- int[] keyFields, int threshold, IPrimitiveIntegerValueProviderFactory valueProviderFactory) {
- super(bufferCache, fileMapProvider, file);
- this.waveletFields = keyFields;
- this.waveletFieldValueProvider = valueProviderFactory.createPrimitiveIntegerValueProvider();
- this.threshold = threshold;
- this.coefficients = new PriorityQueue<>(threshold);
- this.numPages = (int) Math.ceil(threshold * (4 + 8) / (double) bufferCache.getPageSize());
- }
-
- @Override
- public int getNumPages() throws HyracksDataException {
- if (!isActivated) {
- throw new HyracksDataException("The bloom filter is not activated.");
- }
- return numPages;
- }
-
- private void initWaveletSynopsisMetaData() throws HyracksDataException {
- ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), true);
- metaPage.acquireWriteLatch();
- try {
- metaPage.getBuffer().putInt(NUM_PAGES_OFFSET, 0);
- metaPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, 0L);
- } finally {
- metaPage.releaseWriteLatch(true);
- bufferCache.unpin(metaPage);
- }
- }
-
- @Override
- // Adds a new coefficient to the transform, subject to thresholding
- public void addElement(Integer index, Double value) {
- WaveletCoefficient<Integer, Double> newCoeff;
- if (coefficients.size() < threshold)
- newCoeff = new WaveletCoefficient<Integer, Double>(value, 0, index);
- else {
- newCoeff = coefficients.poll();
- newCoeff.setValue(value);
- newCoeff.setIndex(index);
- }
- coefficients.add(newCoeff);
-
- }
-
- @Override
- public IIndexBulkLoader createBuilder() throws HyracksDataException {
- return new SparseTransformBuilder();
- }
-
- public class SparseTransformBuilder implements IIndexBulkLoader {
- private final Stack<WaveletCoefficient<Integer, Double>> avgStack;
- private Pair<Integer, Double> curr;
- private long transformPos;
- private final long domainEnd;
- private int lastLevel;
- private final List<Pair<Integer, Double>> borderTuples;
-
- public SparseTransformBuilder() throws HyracksDataException {
- avgStack = new Stack<>();
- borderTuples = new ArrayList<>(2);
- // initial transform element
- transformPos = Byte.MIN_VALUE;//(int) waveletFieldPointer.minDomainValue();
- domainEnd = Byte.MAX_VALUE;
- lastLevel = 0;
-
- persistWaveletSynopsisMetaData();
- }
-
- private void persistWaveletSynopsisMetaData() throws HyracksDataException {
- ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), false);
- metaPage.acquireWriteLatch();
- try {
- metaPage.getBuffer().putInt(NUM_PAGES_OFFSET, numPages);
- metaPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, threshold);
- } finally {
- metaPage.releaseWriteLatch(true);
- bufferCache.unpin(metaPage);
- }
- }
-
- @Override
- public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
- // if (waveletFields.length > 1)
- // throw new HyracksDataException("Wavelet synopsis does not support composite keys");
- curr = Pair.of((int) waveletFieldValueProvider.getValue(tuple.getFieldData(waveletFields[0]),
- tuple.getFieldStart(waveletFields[0])), 1.0);
- final int maxLevel = waveletFieldValueProvider.maxLevel(tuple.getFieldData(waveletFields[0]),
- tuple.getFieldStart(waveletFields[0]));
-
- while (transformPos <= curr.getKey()) {
- // current position is a left border of dyadic range
- if (curr.getKey() == transformPos) {
- borderTuples.add(Pair.of(0, (double) curr.getValue()));
- return;
- }
- int newLevel = (int) Math.floor(Math.log((double) (curr.getKey()) - transformPos + 1) / Math.log(2));
- long levelRightBorder = 1l << newLevel;
- //add first dummy average
- if (avgStack.isEmpty()) {
- avgStack.push(new WaveletCoefficient<Integer, Double>(0.0, maxLevel, 0));
- lastLevel = newLevel;
- }
-
- // current position is a right border of dyadic range
- if (curr.getKey() == transformPos + levelRightBorder - 1 /*&& curr.position != domainMax*/)
- borderTuples.add(Pair.of((int) (levelRightBorder - 1), (double) curr.getValue()));
-
- WaveletCoefficient<Integer, Double> newCoeff;
- WaveletCoefficient<Integer, Double> topCoeff = avgStack.peek();
- if (newLevel >= lastLevel) {
- topCoeff = computeDyadicRange(lastLevel, maxLevel, topCoeff, borderTuples);
- newCoeff = topCoeff;
- do {
- WaveletCoefficient<Integer, Double> oldCoeff = avgStack.pop();
- //skip first dummy coefficient
- if (oldCoeff.index > 0)
- newCoeff = average(oldCoeff, newCoeff, waveletFieldValueProvider.maxLevel(
- tuple.getFieldData(waveletFields[0]), tuple.getFieldStart(waveletFields[0])));
- } while (!avgStack.isEmpty() && avgStack.peek().level == newCoeff.level);
- } else {
- newCoeff = computeDyadicRange(newLevel, maxLevel, topCoeff, borderTuples);
- topCoeff = newCoeff;
- }
- avgStack.push(newCoeff);
- transformPos += 1l << topCoeff.level;
- lastLevel = newCoeff.level;
-
- borderTuples.clear();
- }
- }
-
- private WaveletCoefficient<Integer, Double> average(WaveletCoefficient<Integer, Double> oldCoeff,
- WaveletCoefficient<Integer, Double> newCoeff, int maxLevel) {
- Integer coeffIdx = oldCoeff.index >> 1;
- addElement(coeffIdx, (oldCoeff.value - newCoeff.value)
- / (2.0 * WaveletCoefficient.getNormalizationCoefficient(maxLevel, oldCoeff.level + 1)));
- WaveletCoefficient<Integer, Double> topCoeff = new WaveletCoefficient<Integer, Double>(
- (oldCoeff.value + newCoeff.value) / 2.0, oldCoeff.level + 1, coeffIdx);
- return topCoeff;
- }
-
- private WaveletCoefficient<Integer, Double> computeDyadicRange(int level, int maxLevel,
- WaveletCoefficient<Integer, Double> topCoeff, List<Pair<Integer, Double>> borderTuples) {
- //short circuit coefficient computation for 0
- if (borderTuples.isEmpty()) {
- Integer coeffIdx = ((topCoeff.index + 1) << (topCoeff.level - level));
- return new WaveletCoefficient<Integer, Double>(0.0, level, coeffIdx);
- }
-
- Map<Integer, Double> newCoefs = new HashMap<>();
- Double avg = 0.0;
- Integer coeffIdx = -1;
- for (int i = 1; i <= level; i++) {
- newCoefs.clear();
- avg = 0.0;
- for (int j = 0; j < borderTuples.size(); j++) {
- Pair<Integer, Double> item = borderTuples.get(j);
- coeffIdx = ((topCoeff.index + 1) << (topCoeff.level - i)) + (item.getKey() >> 1);
- Double newValue = item.getValue() / 2.0;
- Double oldValue = newCoefs.containsKey(coeffIdx) ? newCoefs.get(coeffIdx) : 0;
- if ((item.getKey() & 0x1) == 1) {
- newCoefs.put(coeffIdx, (oldValue - newValue));
- } else {
- newCoefs.put(coeffIdx, (oldValue + newValue));
- }
- avg += newValue;
- borderTuples.set(j, Pair.of(item.getKey() >> 1, newValue));
- }
-
- for (Entry<Integer, Double> e : newCoefs.entrySet())
- addElement(e.getKey(), e.getValue() / WaveletCoefficient.getNormalizationCoefficient(maxLevel, i));
- }
- return new WaveletCoefficient<Integer, Double>(avg, level, coeffIdx);
- }
-
- @Override
- public void end() throws IndexException, HyracksDataException {
- //assert(avgStack.size() == 1);
- if (curr.getKey() != domainEnd) {
-
- }
- WaveletCoefficient<Integer, Double> topCoeff = avgStack.pop();
- topCoeff.index = 0;
- addElement(topCoeff.index, topCoeff.value);
-
- persistStatistics();
- }
-
- private void persistStatistics() throws HyracksDataException {
- List<WaveletCoefficient<Integer, Double>> persistCoefficients = new ArrayList<>(coefficients);
- // sorting coefficients according their keys
- Collections.sort(persistCoefficients, new Comparator<WaveletCoefficient<Integer, Double>>() {
- @Override
- public int compare(WaveletCoefficient<Integer, Double> o1, WaveletCoefficient<Integer, Double> o2) {
- return o1.getKey().compareTo(o2.getKey());
- }
- });
- Iterator<WaveletCoefficient<Integer, Double>> it = persistCoefficients.iterator();
- int currentPageId = 1;
- while (currentPageId <= numPages) {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
- ByteBuffer buffer = page.getBuffer();
- page.acquireWriteLatch();
- try {
- while (it.hasNext() && (buffer.limit() - buffer.position()) >= 4 + 8) {
- WaveletCoefficient<Integer, Double> coeff = it.next();
- buffer.putInt(coeff.getKey());
- buffer.putDouble(coeff.getValue());
- }
- } finally {
- page.releaseWriteLatch(true);
- bufferCache.unpin(page);
- }
- ++currentPageId;
- }
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index cc4e697..a177e3b 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -38,27 +38,34 @@
- <dependencies>
+ <dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
<version>0.2.17-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-bloomfilter</artifactId>
<version>0.2.17-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-common</artifactId>
<version>0.2.17-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
- </dependency>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-statistics</artifactId>
+ <version>0.2.17-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelper.java
index afaa949..5d533ec 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelper.java
@@ -5,7 +5,7 @@ import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -17,18 +17,20 @@ import org.apache.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflo
public abstract class AbstractLSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
protected final boolean collectStatistics;
- protected final IPrimitiveIntegerValueProviderFactory statsValueProviderFactory;
+ protected final IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory;
+ protected final ITypeTraits[] statsFieldTypeTraits;
public AbstractLSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean durable, boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.collectStatistics = collectStatistics;
- this.statsValueProviderFactory = statsValueProviderFactory;
+ this.statsFieldValueProviderFactory = statsFieldValueProviderFactory;
+ this.statsFieldTypeTraits = statsFieldTypeTraits;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelperFactory.java
index 3fa052f..9304ee8 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/AbstractLSMBTreeDataflowHelperFactory.java
@@ -4,7 +4,7 @@ import java.util.Map;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
@@ -16,20 +16,22 @@ public abstract class AbstractLSMBTreeDataflowHelperFactory extends AbstractLSMI
private static final long serialVersionUID = 1L;
protected final boolean collectStatistics;
- protected final IPrimitiveIntegerValueProviderFactory statsValueProviderFactory;
+ protected final IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory;
+ protected final ITypeTraits[] statsFieldTypeTraits;
public AbstractLSMBTreeDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
- boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean durable, boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsKeyValueProviderFactory) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
filterCmpFactories, filterFields, durable);
this.collectStatistics = collectStatistics;
- this.statsValueProviderFactory = statsValueProviderFactory;
+ this.statsFieldValueProviderFactory = statsKeyValueProviderFactory;
+ this.statsFieldTypeTraits = statsFieldTypeTraits;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index 0407257..406c8f0 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -21,9 +21,10 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import java.util.List;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndex;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -42,10 +43,11 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable,
- boolean collectStatistics, IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, false, null, null, null, null, durable, collectStatistics,
- statsValueProviderFactory);
+ ioOpCallbackFactory, false, null, null, null, null, durable, collectStatistics, statsFieldTypeTraits,
+ statsFieldValueProviderFactory);
this.version = version;
}
@@ -53,10 +55,11 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable,
- boolean collectStatistics, IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version, durable, collectStatistics,
- statsValueProviderFactory);
+ statsFieldTypeTraits, statsFieldValueProviderFactory);
}
@Override
@@ -81,7 +84,7 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), getVersion(), durable, collectStatistics,
- statsValueProviderFactory);
+ statsFieldValueProviderFactory);
}
public int getVersion() {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
index c54c826..6f06c72 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -21,8 +21,9 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
@@ -39,10 +40,10 @@ public class ExternalBTreeDataflowHelperFactory extends AbstractLSMBTreeDataflow
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
double bloomFilterFalsePositiveRate, int version, boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ ITypeTraits[] statsFieldTypeTraits, IOrdinalPrimitiveValueProviderFactory statsKeyValueProviderFactory) {
super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null, durable, collectStatistics,
- statsValueProviderFactory);
+ statsFieldTypeTraits, statsKeyValueProviderFactory);
this.version = version;
}
@@ -52,7 +53,7 @@ public class ExternalBTreeDataflowHelperFactory extends AbstractLSMBTreeDataflow
return new ExternalBTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version, durable,
- collectStatistics, statsValueProviderFactory);
+ collectStatistics, statsFieldTypeTraits, statsFieldValueProviderFactory);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index 4b284d0..b5976d8 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -19,9 +19,10 @@
package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndex;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
@@ -39,19 +40,21 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMBTreeDatafl
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] buddyBtreeFields, int version, boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ ITypeTraits[] statsFieldTypeTraits, IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, buddyBtreeFields, version, durable, collectStatistics,
- statsValueProviderFactory);
+ statsFieldTypeTraits, statsFieldValueProviderFactory);
}
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version, boolean durable,
- boolean collectStatistics, IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, null, null, null, durable, collectStatistics, statsValueProviderFactory);
+ ioOpCallbackFactory, null, null, null, durable, collectStatistics, statsFieldTypeTraits,
+ statsFieldValueProviderFactory);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
index 2922897..ca672c2 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -21,8 +21,9 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
@@ -39,10 +40,11 @@ public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMBTre
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
double bloomFilterFalsePositiveRate, int[] buddyBtreeFields, int version, boolean durable,
- boolean collectStatistics, IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null, durable, collectStatistics,
- statsValueProviderFactory);
+ statsFieldTypeTraits, statsFieldValueProviderFactory);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
@@ -53,7 +55,7 @@ public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMBTre
return new ExternalBTreeWithBuddyDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version, durable,
- collectStatistics, statsValueProviderFactory);
+ collectStatistics, statsFieldTypeTraits, statsFieldValueProviderFactory);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index dd2c56f..ffd00ec 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -25,7 +25,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -46,10 +46,12 @@ public class LSMBTreeDataflowHelper extends AbstractLSMBTreeDataflowHelper {
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable,
- boolean collectStatistics, IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ boolean collectStatistics, ITypeTraits[] statsFieldTypeTraits,
+ IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields, durable, collectStatistics, statsValueProviderFactory);
+ filterCmpFactories, btreeFields, filterFields, durable, collectStatistics, statsFieldTypeTraits,
+ statsFieldValueProviderFactory);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
@@ -58,10 +60,10 @@ public class LSMBTreeDataflowHelper extends AbstractLSMBTreeDataflowHelper {
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
int[] btreeFields, int[] filterFields, boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ ITypeTraits[] statsFieldTypeTraits, IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable,
- collectStatistics, statsValueProviderFactory);
+ collectStatistics, statsFieldTypeTraits, statsFieldValueProviderFactory);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
}
@@ -74,6 +76,6 @@ public class LSMBTreeDataflowHelper extends AbstractLSMBTreeDataflowHelper {
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields, durable, collectStatistics, statsValueProviderFactory);
+ btreeFields, filterFields, durable, collectStatistics, statsFieldValueProviderFactory);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 92324a7..e02c4ee 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveIntegerValueProviderFactory;
+import org.apache.hyracks.storage.am.common.api.IOrdinalPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -45,10 +45,11 @@ public class LSMBTreeDataflowHelperFactory extends AbstractLSMBTreeDataflowHelpe
ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
int[] btreeFields, int[] filterFields, boolean durable, boolean collectStatistics,
- IPrimitiveIntegerValueProviderFactory statsValueProviderFactory) {
+ ITypeTraits[] statsFieldTypeTraits, IOrdinalPrimitiveValueProviderFactory statsFieldValueProviderFactory) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
- filterCmpFactories, filterFields, durable, collectStatistics, statsValueProviderFactory);
+ filterCmpFactories, filterFields, durable, collectStatistics, statsFieldTypeTraits,
+ statsFieldValueProviderFactory);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
}
@@ -60,6 +61,7 @@ public class LSMBTreeDataflowHelperFactory extends AbstractLSMBTreeDataflowHelpe
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields, durable, collectStatistics, statsValueProviderFactory);
+ filterCmpFactories, btreeFields, filterFields, durable, collectStatistics, statsFieldTypeTraits,
+ statsFieldValueProviderFactory);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 936c94f..9c04f46 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -47,8 +47,6 @@ import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.statistics.StatisticsFactory;
-import org.apache.hyracks.storage.am.common.statistics.Synopsis;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -66,6 +64,8 @@ import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbac
import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.am.statistics.common.StatisticsFactory;
+import org.apache.hyracks.storage.am.statistics.common.Synopsis;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
/**
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 761b700..96fd6cd 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -57,8 +59,6 @@ import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
-import org.apache.hyracks.storage.am.common.statistics.StatisticsFactory;
-import org.apache.hyracks.storage.am.common.statistics.Synopsis;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -84,11 +84,15 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.am.statistics.common.StatisticsFactory;
+import org.apache.hyracks.storage.am.statistics.common.Synopsis;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
+ private static final Logger LOGGER = Logger.getLogger(LSMBTree.class.getName());
+
// For creating BTree's used in flush and merge.
protected final LSMBTreeDiskComponentFactory componentFactory;
// For creating BTree's used in bulk load. Different from diskBTreeFactory
@@ -99,7 +103,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
protected final ITreeIndexFrameFactory insertLeafFrameFactory;
protected final ITreeIndexFrameFactory deleteLeafFrameFactory;
protected final IBinaryComparatorFactory[] cmpFactories;
- protected final boolean collectStatistics;
+ protected boolean collectStatistics;
private final boolean needKeyDupCheck;
private final int[] btreeFields;
@@ -234,9 +238,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
BloomFilter bloomFilter = component.getBloomFilter();
btree.deactivate();
bloomFilter.deactivate();
- Synopsis stats = component.getStatistics();
- if (stats != null)
- stats.deactivate();
+ if (collectStatistics)
+ component.getStatistics().deactivate();
}
for (ILSMComponent c : memoryComponents) {
LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
@@ -289,8 +292,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
component.getBTree().destroy();
component.getBloomFilter().deactivate();
component.getBloomFilter().destroy();
- Synopsis stats = component.getStatistics();
- if (stats != null) {
+ if (collectStatistics) {
+ Synopsis stats = component.getStatistics();
stats.deactivate();
stats.destroy();
}
@@ -506,8 +509,14 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
IIndexBulkLoader builder = component.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(),
bloomFilterSpec.getNumBucketsPerElements());
IIndexBulkLoader statsBuilder = null;
- if (collectStatistics)
- statsBuilder = component.getStatistics().createBuilder();
+ try {
+ if (collectStatistics)
+ statsBuilder = component.getStatistics().createBuilder();
+ } catch (HyracksDataException e) {
+ if (LOGGER.isLoggable(Level.FINE))
+ LOGGER.fine("Statistics collection failed: " + e.getMessage());
+ collectStatistics = false;
+ }
IIndexCursor scanCursor = accessor.createSearchCursor(false);
accessor.search(scanCursor, nullPred);
@@ -586,8 +595,14 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
IIndexBulkLoader bloomFilterBuilder = mergedComponent.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
IIndexBulkLoader statsBuilder = null;
- if (collectStatistics)
- statsBuilder = mergedComponent.getStatistics().createBuilder();
+ try {
+ if (collectStatistics)
+ statsBuilder = mergedComponent.getStatistics().createBuilder();
+ } catch (HyracksDataException e) {
+ if (LOGGER.isLoggable(Level.FINE))
+ LOGGER.fine("Statistics collection failed: " + e.getMessage());
+ collectStatistics = false;
+ }
try {
while (cursor.hasNext()) {
cursor.next();
@@ -627,9 +642,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
if (createComponent) {
component.getBTree().create();
component.getBloomFilter().create();
+ if (createStatistics)
+ component.getStatistics().create();
}
- if (createStatistics)
- component.getStatistics().create();
// BTree will be closed during cleanup of merge().
component.getBTree().activate();
component.getBloomFilter().activate();
@@ -708,8 +723,14 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
bloomFilterFalsePositiveRate);
bloomFilterBuilder = ((AbstractDiskLSMComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
- if (collectStatistics)
- statisticsBuilder = ((LSMBTreeDiskComponent) component).getStatistics().createBuilder();
+ try {
+ if (collectStatistics)
+ statisticsBuilder = ((LSMBTreeDiskComponent) component).getStatistics().createBuilder();
+ } catch (HyracksDataException e) {
+ if (LOGGER.isLoggable(Level.INFO))
+ LOGGER.info("Statistics collection failed: " + e.getMessage());
+ collectStatistics = false;
+ }
if (filterFields != null) {
indexTuple = new PermutingTupleReference(btreeFields);
@@ -759,12 +780,14 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
bloomFilterBuilder.end();
endedBloomFilterLoad = true;
}
- if (collectStatistics)
- statisticsBuilder.end();
((LSMBTreeDiskComponent) component).getBTree().deactivate();
((LSMBTreeDiskComponent) component).getBTree().destroy();
((AbstractDiskLSMComponent) component).getBloomFilter().deactivate();
((AbstractDiskLSMComponent) component).getBloomFilter().destroy();
+ if (((LSMBTreeDiskComponent) component).getStatistics() != null) {
+ ((LSMBTreeDiskComponent) component).getStatistics().deactivate();
+ ((LSMBTreeDiskComponent) component).getStatistics().destroy();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index 8746cff..45f8294 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -21,9 +21,9 @@ package org.apache.hyracks.storage.am.lsm.btree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.statistics.Synopsis;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
+import org.apache.hyracks.storage.am.statistics.common.Synopsis;
public class LSMBTreeDiskComponent extends AbstractDiskLSMComponent {
private final BTree btree;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/e05a1921/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
index d3d3556..ea25efc 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
@@ -23,12 +23,12 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.statistics.StatisticsFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.am.statistics.common.StatisticsFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
public class LSMBTreeDiskComponentFactory implements ILSMComponentFactory {