You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/23 03:49:44 UTC
[7/8] kylin git commit: KYLIN-2195 All code changes, ready for test
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index db02f34..db9fc80 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -1,246 +1,246 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable<ByteArray>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public static final ByteArray EMPTY = new ImmutableByteArray();
-
- public static ByteArray allocate(int length) {
- return new ByteArray(new byte[length]);
- }
-
- public static ByteArray copyOf(byte[] array, int offset, int length) {
- byte[] space = new byte[length];
- System.arraycopy(array, offset, space, 0, length);
- return new ByteArray(space, 0, length);
- }
-
- // ============================================================================
-
- private byte[] data;
- private int offset;
- private int length;
-
- public ByteArray() {
- this(null, 0, 0);
- }
-
- public ByteArray(int capacity) {
- this(new byte[capacity], 0, capacity);
- }
-
- public ByteArray(byte[] data) {
- this(data, 0, data == null ? 0 : data.length);
- }
-
- public ByteArray(byte[] data, int offset, int length) {
- this.data = data;
- this.offset = offset;
- this.length = length;
- }
-
- public byte[] array() {
- return data;
- }
-
- public int offset() {
- return offset;
- }
-
- public int length() {
- return length;
- }
-
- public void set(byte[] array) {
- set(array, 0, array.length);
- }
-
- public void set(byte[] array, int offset, int length) {
- this.data = array;
- this.offset = offset;
- this.length = length;
- }
-
- public void set(ByteArray o) {
- set(o.data, o.offset, o.length);
- }
-
- public void set(int offset, int length) {
- this.offset = offset;
- this.length = length;
- }
-
- public void setLength(int length) {
- this.length = length;
- }
-
- public ByteArray copy() {
- ByteArray copy;
- if (data != null) {
- copy = new ByteArray(length);
- } else {
- copy = new ByteArray(null);
- }
- copy.copyFrom(this);
- return copy;
- }
-
- //notice this will have a length header
- public void exportData(ByteBuffer out) {
- BytesUtil.writeByteArray(this.data, this.offset, this.length, out);
- }
-
- public static ByteArray importData(ByteBuffer in) {
- byte[] bytes = BytesUtil.readByteArray(in);
- return new ByteArray(bytes);
- }
-
- public void copyFrom(ByteArray other) {
- if (other.data != null) {
- System.arraycopy(other.array(), other.offset, data, offset, other.length);
- }
- this.length = other.length;
- }
-
- public ByteBuffer asBuffer() {
- if (data == null)
- return null;
- else if (offset == 0 && length == data.length)
- return ByteBuffer.wrap(data);
- else
- return ByteBuffer.wrap(data, offset, length).slice();
- }
-
- public byte[] toBytes() {
- return Bytes.copy(this.array(), this.offset(), this.length());
- }
-
- @Override
- public int hashCode() {
- if (data == null) {
- return 0;
- } else {
- if (length <= Bytes.SIZEOF_LONG && length > 0) {
- // to avoid hash collision of byte arrays those are converted from nearby integers/longs, which is the case for kylin dictionary
- long value = BytesUtil.readLong(data, offset, length);
- return (int) (value ^ (value >>> 32));
- }
- return Bytes.hashCode(data, offset, length);
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ByteArray o = (ByteArray) obj;
- if (this.data == null && o.data == null)
- return true;
- else if (this.data == null || o.data == null)
- return false;
- else
- return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length);
- }
-
- @Override
- public int compareTo(ByteArray o) {
- if (this.data == null && o.data == null)
- return 0;
- else if (this.data == null)
- return -1;
- else if (o.data == null)
- return 1;
- else
- return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length);
- }
-
- public String toReadableText() {
- if (data == null) {
- return null;
- } else {
- return BytesUtil.toHex(data, offset, length);
- }
- }
-
- @Override
- public String toString() {
- if (data == null)
- return null;
- else
- return Bytes.toStringBinary(data, offset, length);
- }
-
- // ============================================================================
-
- public static class ImmutableByteArray extends ByteArray {
-
- private static final long serialVersionUID = 1L;
-
- public ImmutableByteArray() {
- super();
- }
-
- public ImmutableByteArray(byte[] data, int offset, int length) {
- super(data, offset, length);
- }
-
- public ImmutableByteArray(byte[] data) {
- super(data);
- }
-
- @Override
- public void set(byte[] array) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void set(byte[] array, int offset, int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void set(ByteArray o) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setLength(int length) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void copyFrom(ByteArray other) {
- throw new UnsupportedOperationException();
- }
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+/**
+ * @author yangli9
+ */
+public class ByteArray implements Comparable<ByteArray>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final ByteArray EMPTY = new ImmutableByteArray();
+
+ public static ByteArray allocate(int length) {
+ return new ByteArray(new byte[length]);
+ }
+
+ public static ByteArray copyOf(byte[] array, int offset, int length) {
+ byte[] space = new byte[length];
+ System.arraycopy(array, offset, space, 0, length);
+ return new ByteArray(space, 0, length);
+ }
+
+ // ============================================================================
+
+ private byte[] data;
+ private int offset;
+ private int length;
+
+ public ByteArray() {
+ this(null, 0, 0);
+ }
+
+ public ByteArray(int capacity) {
+ this(new byte[capacity], 0, capacity);
+ }
+
+ public ByteArray(byte[] data) {
+ this(data, 0, data == null ? 0 : data.length);
+ }
+
+ public ByteArray(byte[] data, int offset, int length) {
+ this.data = data;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public byte[] array() {
+ return data;
+ }
+
+ public int offset() {
+ return offset;
+ }
+
+ public int length() {
+ return length;
+ }
+
+ public void set(byte[] array) {
+ set(array, 0, array.length);
+ }
+
+ public void set(byte[] array, int offset, int length) {
+ this.data = array;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void set(ByteArray o) {
+ set(o.data, o.offset, o.length);
+ }
+
+ public void set(int offset, int length) {
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public void setLength(int length) {
+ this.length = length;
+ }
+
+ public ByteArray copy() {
+ ByteArray copy;
+ if (data != null) {
+ copy = new ByteArray(length);
+ } else {
+ copy = new ByteArray(null);
+ }
+ copy.copyFrom(this);
+ return copy;
+ }
+
+ //notice this will have a length header
+ public void exportData(ByteBuffer out) {
+ BytesUtil.writeByteArray(this.data, this.offset, this.length, out);
+ }
+
+ public static ByteArray importData(ByteBuffer in) {
+ byte[] bytes = BytesUtil.readByteArray(in);
+ return new ByteArray(bytes);
+ }
+
+ public void copyFrom(ByteArray other) {
+ if (other.data != null) {
+ System.arraycopy(other.array(), other.offset, data, offset, other.length);
+ }
+ this.length = other.length;
+ }
+
+ public ByteBuffer asBuffer() {
+ if (data == null)
+ return null;
+ else if (offset == 0 && length == data.length)
+ return ByteBuffer.wrap(data);
+ else
+ return ByteBuffer.wrap(data, offset, length).slice();
+ }
+
+ public byte[] toBytes() {
+ return Bytes.copy(this.array(), this.offset(), this.length());
+ }
+
+ @Override
+ public int hashCode() {
+ if (data == null) {
+ return 0;
+ } else {
+ if (length <= Bytes.SIZEOF_LONG && length > 0) {
+ // to avoid hash collision of byte arrays those are converted from nearby integers/longs, which is the case for kylin dictionary
+ long value = BytesUtil.readLong(data, offset, length);
+ return (int) (value ^ (value >>> 32));
+ }
+ return Bytes.hashCode(data, offset, length);
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ByteArray o = (ByteArray) obj;
+ if (this.data == null && o.data == null)
+ return true;
+ else if (this.data == null || o.data == null)
+ return false;
+ else
+ return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length);
+ }
+
+ @Override
+ public int compareTo(ByteArray o) {
+ if (this.data == null && o.data == null)
+ return 0;
+ else if (this.data == null)
+ return -1;
+ else if (o.data == null)
+ return 1;
+ else
+ return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length);
+ }
+
+ public String toReadableText() {
+ if (data == null) {
+ return null;
+ } else {
+ return BytesUtil.toHex(data, offset, length);
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (data == null)
+ return null;
+ else
+ return Bytes.toStringBinary(data, offset, length);
+ }
+
+ // ============================================================================
+
+ public static class ImmutableByteArray extends ByteArray {
+
+ private static final long serialVersionUID = 1L;
+
+ public ImmutableByteArray() {
+ super();
+ }
+
+ public ImmutableByteArray(byte[] data, int offset, int length) {
+ super(data, offset, length);
+ }
+
+ public ImmutableByteArray(byte[] data) {
+ super(data);
+ }
+
+ @Override
+ public void set(byte[] array) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void set(byte[] array, int offset, int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void set(ByteArray o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setLength(int length) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void copyFrom(ByteArray other) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
index 26342f5..699d569 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
@@ -1,35 +1,35 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- *
- */
-public interface BytesSerializer<T> {
-
- int SERIALIZE_BUFFER_SIZE = 65536;
-
- void serialize(T value, ByteBuffer out);
-
- T deserialize(ByteBuffer in);
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @author yangli9
+ *
+ */
+public interface BytesSerializer<T> {
+
+ int SERIALIZE_BUFFER_SIZE = 65536;
+
+ void serialize(T value, ByteBuffer out);
+
+ T deserialize(ByteBuffer in);
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
index 33d1d05..c644890 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
@@ -1,125 +1,125 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class BytesSplitter {
- private static final Logger logger = LoggerFactory.getLogger(BytesSplitter.class);
-
- private static final int[] COMMON_DELIMS = new int[] { "\177".codePointAt(0), "|".codePointAt(0), "\t".codePointAt(0), ",".codePointAt(0) };
-
- private SplittedBytes[] splitBuffers;
- private int bufferSize;
-
- public SplittedBytes[] getSplitBuffers() {
- return splitBuffers;
- }
-
- public SplittedBytes getSplitBuffer(int index) {
- return splitBuffers[index];
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public BytesSplitter(int splitLen, int bytesLen) {
- this.splitBuffers = new SplittedBytes[splitLen];
- for (int i = 0; i < splitLen; i++) {
- this.splitBuffers[i] = new SplittedBytes(bytesLen);
- }
- this.bufferSize = 0;
- }
-
- public int split(byte[] bytes, int byteLen, byte delimiter) {
- this.bufferSize = 0;
- int offset = 0;
- int length = 0;
- for (int i = 0; i < byteLen; i++) {
- if (bytes[i] == delimiter) {
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- if (length > split.value.length) {
- length = split.value.length;
- }
- System.arraycopy(bytes, offset, split.value, 0, length);
- split.length = length;
- offset = i + 1;
- length = 0;
- } else {
- length++;
- }
- }
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- if (length > split.value.length) {
- length = split.value.length;
- }
- System.arraycopy(bytes, offset, split.value, 0, length);
- split.length = length;
-
- return bufferSize;
- }
-
- public void setBuffers(byte[][] buffers) {
- for (int i = 0; i < buffers.length; i++) {
- splitBuffers[i].value = buffers[i];
- splitBuffers[i].length = buffers[i].length;
- }
- this.bufferSize = buffers.length;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for (int i = 0; i < bufferSize; i++) {
- if (i > 0)
- buf.append(", ");
-
- buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length));
- }
- return buf.toString();
- }
-
- public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) {
- List<String> splitStrings = new ArrayList<String>();
- int splitOffset = 0;
- int splitLength = 0;
- for (int i = offset; i < bytes.length; i++) {
- if (bytes[i] == delimiter) {
- String str = Bytes.toString(bytes, splitOffset, splitLength);
- splitStrings.add(str);
- splitOffset = i + 1;
- splitLength = 0;
- } else {
- splitLength++;
- }
- }
- String str = Bytes.toString(bytes, splitOffset, splitLength);
- splitStrings.add(str);
- return splitStrings;
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class BytesSplitter {
+ private static final Logger logger = LoggerFactory.getLogger(BytesSplitter.class);
+
+ private static final int[] COMMON_DELIMS = new int[] { "\177".codePointAt(0), "|".codePointAt(0), "\t".codePointAt(0), ",".codePointAt(0) };
+
+ private SplittedBytes[] splitBuffers;
+ private int bufferSize;
+
+ public SplittedBytes[] getSplitBuffers() {
+ return splitBuffers;
+ }
+
+ public SplittedBytes getSplitBuffer(int index) {
+ return splitBuffers[index];
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public BytesSplitter(int splitLen, int bytesLen) {
+ this.splitBuffers = new SplittedBytes[splitLen];
+ for (int i = 0; i < splitLen; i++) {
+ this.splitBuffers[i] = new SplittedBytes(bytesLen);
+ }
+ this.bufferSize = 0;
+ }
+
+ public int split(byte[] bytes, int byteLen, byte delimiter) {
+ this.bufferSize = 0;
+ int offset = 0;
+ int length = 0;
+ for (int i = 0; i < byteLen; i++) {
+ if (bytes[i] == delimiter) {
+ SplittedBytes split = this.splitBuffers[this.bufferSize++];
+ if (length > split.value.length) {
+ length = split.value.length;
+ }
+ System.arraycopy(bytes, offset, split.value, 0, length);
+ split.length = length;
+ offset = i + 1;
+ length = 0;
+ } else {
+ length++;
+ }
+ }
+ SplittedBytes split = this.splitBuffers[this.bufferSize++];
+ if (length > split.value.length) {
+ length = split.value.length;
+ }
+ System.arraycopy(bytes, offset, split.value, 0, length);
+ split.length = length;
+
+ return bufferSize;
+ }
+
+ public void setBuffers(byte[][] buffers) {
+ for (int i = 0; i < buffers.length; i++) {
+ splitBuffers[i].value = buffers[i];
+ splitBuffers[i].length = buffers[i].length;
+ }
+ this.bufferSize = buffers.length;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("[");
+ for (int i = 0; i < bufferSize; i++) {
+ if (i > 0)
+ buf.append(", ");
+
+ buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length));
+ }
+ return buf.toString();
+ }
+
+ public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) {
+ List<String> splitStrings = new ArrayList<String>();
+ int splitOffset = 0;
+ int splitLength = 0;
+ for (int i = offset; i < bytes.length; i++) {
+ if (bytes[i] == delimiter) {
+ String str = Bytes.toString(bytes, splitOffset, splitLength);
+ splitStrings.add(str);
+ splitOffset = i + 1;
+ splitLength = 0;
+ } else {
+ splitLength++;
+ }
+ }
+ String str = Bytes.toString(bytes, splitOffset, splitLength);
+ splitStrings.add(str);
+ return splitStrings;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index aa25c22..f97b609 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -1,149 +1,149 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.apache.commons.io.FileUtils;
-
-/**
- * @author yangli9
- */
-public class CliCommandExecutor {
-
- private String remoteHost;
- private int port;
- private String remoteUser;
- private String remotePwd;
- private int remoteTimeoutSeconds = 3600;
-
- public CliCommandExecutor() {
- }
-
- public void setRunAtRemote(String host, int port, String user, String pwd) {
- this.remoteHost = host;
- this.port = port;
- this.remoteUser = user;
- this.remotePwd = pwd;
- }
-
- public void setRunAtLocal() {
- this.remoteHost = null;
- this.remoteUser = null;
- this.remotePwd = null;
- }
-
- public void copyFile(String localFile, String destDir) throws IOException {
- if (remoteHost == null)
- copyNative(localFile, destDir);
- else
- copyRemote(localFile, destDir);
- }
-
- private void copyNative(String localFile, String destDir) throws IOException {
- File src = new File(localFile);
- File dest = new File(destDir, src.getName());
- FileUtils.copyFile(src, dest);
- }
-
- private void copyRemote(String localFile, String destDir) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd);
- try {
- ssh.scpFileToRemote(localFile, destDir);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- public Pair<Integer, String> execute(String command) throws IOException {
- return execute(command, new SoutLogger());
- }
-
- public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException {
- Pair<Integer, String> r;
- if (remoteHost == null) {
- r = runNativeCommand(command, logAppender);
- } else {
- r = runRemoteCommand(command, logAppender);
- }
-
- if (r.getFirst() != 0)
- throw new IOException("OS command error exit with " + r.getFirst() //
- + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") //
- + " -- " + command + "\n" + r.getSecond());
-
- return r;
- }
-
- private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd);
-
- SSHClientOutput sshOutput;
- try {
- sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender);
- int exitCode = sshOutput.getExitCode();
- String output = sshOutput.getText();
- return Pair.newPair(exitCode, output);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException {
- String[] cmd = new String[3];
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Windows")) {
- cmd[0] = "cmd.exe";
- cmd[1] = "/C";
- } else {
- cmd[0] = "/bin/bash";
- cmd[1] = "-c";
- }
- cmd[2] = command;
-
- ProcessBuilder builder = new ProcessBuilder(cmd);
- builder.redirectErrorStream(true);
- Process proc = builder.start();
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
- String line;
- StringBuilder result = new StringBuilder();
- while ((line = reader.readLine()) != null) {
- result.append(line).append('\n');
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
-
- try {
- int exitCode = proc.waitFor();
- return Pair.newPair(exitCode, result.toString());
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.commons.io.FileUtils;
+
+/**
+ * @author yangli9
+ */
+public class CliCommandExecutor {
+
+ private String remoteHost;
+ private int port;
+ private String remoteUser;
+ private String remotePwd;
+ private int remoteTimeoutSeconds = 3600;
+
+ public CliCommandExecutor() {
+ }
+
+ public void setRunAtRemote(String host, int port, String user, String pwd) {
+ this.remoteHost = host;
+ this.port = port;
+ this.remoteUser = user;
+ this.remotePwd = pwd;
+ }
+
+ public void setRunAtLocal() {
+ this.remoteHost = null;
+ this.remoteUser = null;
+ this.remotePwd = null;
+ }
+
+ public void copyFile(String localFile, String destDir) throws IOException {
+ if (remoteHost == null)
+ copyNative(localFile, destDir);
+ else
+ copyRemote(localFile, destDir);
+ }
+
+ private void copyNative(String localFile, String destDir) throws IOException {
+ File src = new File(localFile);
+ File dest = new File(destDir, src.getName());
+ FileUtils.copyFile(src, dest);
+ }
+
+ private void copyRemote(String localFile, String destDir) throws IOException {
+ SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd);
+ try {
+ ssh.scpFileToRemote(localFile, destDir);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ public Pair<Integer, String> execute(String command) throws IOException {
+ return execute(command, new SoutLogger());
+ }
+
+ public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException {
+ Pair<Integer, String> r;
+ if (remoteHost == null) {
+ r = runNativeCommand(command, logAppender);
+ } else {
+ r = runRemoteCommand(command, logAppender);
+ }
+
+ if (r.getFirst() != 0)
+ throw new IOException("OS command error exit with " + r.getFirst() //
+ + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") //
+ + " -- " + command + "\n" + r.getSecond());
+
+ return r;
+ }
+
+ private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException {
+ SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd);
+
+ SSHClientOutput sshOutput;
+ try {
+ sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender);
+ int exitCode = sshOutput.getExitCode();
+ String output = sshOutput.getText();
+ return Pair.newPair(exitCode, output);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException {
+ String[] cmd = new String[3];
+ String osName = System.getProperty("os.name");
+ if (osName.startsWith("Windows")) {
+ cmd[0] = "cmd.exe";
+ cmd[1] = "/C";
+ } else {
+ cmd[0] = "/bin/bash";
+ cmd[1] = "-c";
+ }
+ cmd[2] = command;
+
+ ProcessBuilder builder = new ProcessBuilder(cmd);
+ builder.redirectErrorStream(true);
+ Process proc = builder.start();
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+ String line;
+ StringBuilder result = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ result.append(line).append('\n');
+ if (logAppender != null) {
+ logAppender.log(line);
+ }
+ }
+
+ try {
+ int exitCode = proc.waitFor();
+ return Pair.newPair(exitCode, result.toString());
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index ec0eaac..5793967 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -79,7 +79,7 @@ public class MailService {
if (!enabled) {
logger.info("Email service is disabled; this mail will not be delivered: " + subject);
- logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties");
+ logger.info("To enable mail service, set 'kylin.job.notification-enabled=true' in kylin.properties");
return false;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java b/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java
index 8d728c8..f3ee411 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java
@@ -1,67 +1,67 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-public class ShardingHash {
-
- static HashFunction hashFunc = Hashing.murmur3_128();
-
- public static short getShard(int integerValue, int totalShards) {
- if (totalShards <= 1) {
- return 0;
- }
- long hash = hashFunc.hashInt(integerValue).asLong();
- return _getShard(hash, totalShards);
- }
-
- public static short getShard(long longValue, int totalShards) {
- if (totalShards <= 1) {
- return 0;
- }
- long hash = hashFunc.hashLong(longValue).asLong();
- return _getShard(hash, totalShards);
- }
-
- public static short getShard(byte[] byteValues, int offset, int length, int totalShards) {
- if (totalShards <= 1) {
- return 0;
- }
-
- long hash = hashFunc.hashBytes(byteValues, offset, length).asLong();
- return _getShard(hash, totalShards);
- }
-
- public static short normalize(short cuboidShardBase, short shardOffset, int totalShards) {
- if (totalShards <= 1) {
- return 0;
- }
- return (short) ((cuboidShardBase + shardOffset) % totalShards);
- }
-
- private static short _getShard(long hash, int totalShard) {
- long x = hash % totalShard;
- if (x < 0) {
- x += totalShard;
- }
- return (short) x;
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+public class ShardingHash {
+
+ static HashFunction hashFunc = Hashing.murmur3_128();
+
+ public static short getShard(int integerValue, int totalShards) {
+ if (totalShards <= 1) {
+ return 0;
+ }
+ long hash = hashFunc.hashInt(integerValue).asLong();
+ return _getShard(hash, totalShards);
+ }
+
+ public static short getShard(long longValue, int totalShards) {
+ if (totalShards <= 1) {
+ return 0;
+ }
+ long hash = hashFunc.hashLong(longValue).asLong();
+ return _getShard(hash, totalShards);
+ }
+
+ public static short getShard(byte[] byteValues, int offset, int length, int totalShards) {
+ if (totalShards <= 1) {
+ return 0;
+ }
+
+ long hash = hashFunc.hashBytes(byteValues, offset, length).asLong();
+ return _getShard(hash, totalShards);
+ }
+
+ public static short normalize(short cuboidShardBase, short shardOffset, int totalShards) {
+ if (totalShards <= 1) {
+ return 0;
+ }
+ return (short) ((cuboidShardBase + shardOffset) % totalShards);
+ }
+
+ private static short _getShard(long hash, int totalShard) {
+ long x = hash % totalShard;
+ if (x < 0) {
+ x += totalShard;
+ }
+ return (short) x;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/resources/kylin-backward-compatibility.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-backward-compatibility.properties b/core-common/src/main/resources/kylin-backward-compatibility.properties
index 70cc3f6..563ec26 100644
--- a/core-common/src/main/resources/kylin-backward-compatibility.properties
+++ b/core-common/src/main/resources/kylin-backward-compatibility.properties
@@ -1,183 +1,183 @@
-#### ENV ###
-#
-#deploy.env=kylin.env
-#kylin.hdfs.working.dir=kylin.env.hdfs-working-dir
-#
-#
-#### METADATA ###
-#
-#kylin.metadata.url=kylin.metadata.url
-#kylin.realization.providers=kylin.metadata.realization-providers
-#kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
-#kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types.
-#
-#
-#### Dictionary ###
-#
-#kylin.table.snapshot.max_mb=kylin.snapshot.max-mb
-#kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry
-#kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb
-#kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry
-#kylin.dict.growing.enabled=kylin.dictionary.growing-enabled
-#kylin.dict.append.entry.size=kylin.dictionary.append-entry-size
-#kylin.dict.append.cache.size=kylin.dictionary.append-cache-size
-#
-#
-#### CUBE ###
-#
-#kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio
-#kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio
-#kylin.cube.algorithm=kylin.cube.algorithm
-#kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold
-#kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit
-#kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size
-#kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination
-#kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid
-#kylin.cube.building.segment.max=kylin.cube.max-building-segments
-#
-#
-#### JOB ###
-#
-#kylin.job.log.dir=kylin.job.log-dir
-#kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
-#kylin.job.allow.empty.segment=kylin.job.allow-empty-segment
-#kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs
-#kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage
-#kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision
-#kylin.job.dependency.filterlist=kylin.job.dependency-filter-list
-#kylin.job.retry=kylin.job.retry
-#kylin.job.controller.lock=kylin.job.lock
-#kylin.scheduler.=kylin.job.scheduler.provider.
-#kylin.enable.scheduler=kylin.job.scheduler.default
-#
-#mail.enabled=kylin.job.notification-enabled
-#mail.host=kylin.job.notification-mail-host
-#mail.username=kylin.job.notification-mail-username
-#mail.password=kylin.job.notification-mail-password
-#mail.sender=kylin.job.notification-mail-sender
-#kylin.job.admin.dls=kylin.job.notification-admin-emails
-#
-##for dev
-#kylin.job.run.as.remote.cmd=kylin.job.use-remote-cli
-#kylin.job.remote.cli.port=kylin.job.remote-cli-port
-#kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname
-#kylin.job.remote.cli.username=kylin.job.remote-cli-username
-#kylin.job.remote.cli.password=kylin.job.remote-cli-password
-#
-#
-#### ENGINE ###
-#
-#kylin.cube.engine.=kylin.engine.provider.
-#kylin.default.cube.engine=kylin.engine.default
-#
-#kylin.job.mr.lib.dir=kylin.engine.mr.lib-dir
-#kylin.job.mr.config.override.=kylin.engine.mr.config-override.
-#kylin.job.jar=kylin.engine.mr.job-jar
-#kylin.job.mapreduce.default.reduce.input.mb=kylin.engine.mr.reduce-input-mb
-#kylin.job.mapreduce.default.reduce.count.ratio=kylin.engine.mr.reduce-count-ratio
-#kylin.job.mapreduce.min.reducer.number=kylin.engine.mr.min-reducer-number
-#kylin.job.mapreduce.max.reducer.number=kylin.engine.mr.max-reducer-number
-#kylin.job.mapreduce.mapper.input.rows=kylin.engine.mr.mapper-input-rows
-#kylin.job.uhc.reducer.count=kylin.engine.mr.uhc-reducer-count
-#kylin.job.yarn.app.rest.check.interval.seconds=kylin.engine.mr.yarn-check-interval-seconds
-#
-#kylin.job.jar.spark=kylin.engine.spark.job-jar
-#kylin.spark.home=kylin.engine.spark.spark-home
-#kylin.spark.master=kylin.engine.spark.spark-master
-#
-##deprecated
-#kylin.job.yarn.app.rest.check.status.url=kylin.engine.mr.yarn-check-status-url
-#
-#
-#### SOURCE ###
-#
-#kylin.source.engine.=kylin.source.provider.
-#kylin.hive.config.override.=kylin.source.hive.config-override.
-#kylin.hive.keep.flat.table=kylin.source.hive.keep-flat-table
-#kylin.job.hive.database.for.intermediatetable=kylin.source.hive.database-for-flat-table
-#kylin.job.hive.intermediatetable.redistribute.enabled=kylin.source.hive.redistribute-flat-table
-#kylin.hive.client=kylin.source.hive.client
-#kylin.hive.beeline.params=kylin.source.hive.beeline-params
-#kylin.hive.create.flat.table.method=kylin.source.hive.create-flat-table-method
-#
-##deprecated
-#hive.url=kylin.source.hive.url
-#hive.user=kylin.source.hive.user
-#hive.password=kylin.source.hive.password
-#hive.table.location.=kylin.source.hive.table-location.
-#
-#
-#### STORAGE ###
-#
-#kylin.storage.url=kylin.storage.url
-#kylin.storage.engine.=kylin.storage.provider.
-#kylin.default.storage.engine=kylin.storage.default
-#
-#kylin.hbase.cluster.fs=kylin.storage.hbase.cluster-fs
-#kylin.hbase.cluster.hdfs.config.file=kylin.storage.hbase.cluster-hdfs-config-file
-#kylin.coprocessor.local.jar=kylin.storage.hbase.coprocessor-local-jar
-#kylin.hbase.region.count.min=kylin.storage.hbase.min-region-count
-#kylin.hbase.region.count.max=kylin.storage.hbase.max-region-count
-#kylin.hbase.hfile.size.gb=kylin.storage.hbase.hfile-size-gb
-#kylin.query.run.local.coprocessor=kylin.storage.hbase.run-local-coprocessor
-#kylin.query.coprocessor.mem.gb=kylin.storage.hbase.coprocessor-mem-gb
-#kylin.query.coprocessor.timeout.seconds=kylin.storage.hbase.coprocessor-timeout-seconds
-#kylin.query.scan.fuzzykey.max=kylin.storage.hbase.max-fuzzykey-scan
-#kylin.query.storage.visit.scanrange.max=kylin.storage.hbase.max-visit-scanrange
-#kylin.query.storage.default.gtstorage=kylin.storage.hbase.gtstorage
-#kylin.hbase.scan.cache_rows=kylin.storage.hbase.scan-cache-rows
-#kylin.hbase.region.cut=kylin.storage.hbase.region-cut-gb
-#kylin.hbase.scan.max_result_size=kylin.storage.hbase.max-scan-result-bytes
-#kylin.hbase.default.compression.codec=kylin.storage.hbase.compression-codec
-#kylin.hbase.default.encoding=kylin.storage.hbase.rowkey-encoding
-#kylin.hbase.default.block.size=kylin.storage.hbase.block-size-bytes
-#kylin.hbase.small.family.block.size=kylin.storage.hbase.small-family-block-size-bytes
-#kylin.owner=kylin.storage.hbase.owner-tag
-#kylin.query.endpoint.compression.result=kylin.storage.hbase.endpoint-compress-result
-#kylin.query.hbase.hconnection.threads.max=kylin.storage.hbase.max-hconnection-threads
-#kylin.query.hbase.hconnection.threads.core=kylin.storage.hbase.core-hconnection-threads
-#kylin.query.hbase.hconnection.threads.alive.seconds=kylin.storage.hbase.hconnection-threads-alive-seconds
-#
-#
-#### QUERY ###
-#
-#kylin.query.pushdown.limit.max=kylin.query.max-limit-pushdown
-#kylin.query.scan.threshold=kylin.query.scan-threshold
-#kylin.query.filter.derived_in.max=kylin.query.derived-filter-translation-threshold
-#kylin.query.badquery.stacktrace.depth=kylin.query.badquery-stacktrace-depth
-#kylin.query.badquery.history.num=kylin.query.badquery-history-number
-#kylin.query.badquery.alerting.seconds=kylin.query.badquery-alerting-seconds
-#kylin.query.badquery.detect.interval.seconds=kylin.query.badquery-detect-interval
-#kylin.query.badquery.persistent.enable=kylin.query.badquery-persistent-enabled
-#kylin.query.transformers=kylin.query.transformers
-#kylin.query.cache.enabled=kylin.query.cache-enabled
-#kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration
-#kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count
-#kylin.query.mem.budget=kylin.query.memory-budget
-#kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function
-#kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct
-#kylin.query.security.enabled=kylin.query.security-enabled
-#kylin.query.access.controller=kylin.query.access-controller
-#kylin.query.udf.=kylin.query.udf.
-#
-#
-#### SERVER ###
-#
-#kylin.init.tasks=kylin.server.init-tasks
-#kylin.server.mode=kylin.server.mode
-#kylin.cluster.name=kylin.server.cluster-name
-#kylin.rest.servers=kylin.server.cluster-servers
-#kylin.rest.workers.per.server=kylin.server.sequence-sql.workers-per-server
-#kylin.query.sequence.expire.time=kylin.server.sequence-sql.expire-time
-#kylin.query.metrics.enabled=kylin.server.query-metrics-enabled
-#kylin.query.metrics.percentiles.intervals=kylin.server.query-metrics-percentiles-intervals
-#
-#
-#### WEB ###
-#
-#kylin.rest.timezone=kylin.web.timezone
-#crossdomain.enable=kylin.web.cross-domain-enabled
+### ENV ###
+
+deploy.env=kylin.env
+kylin.hdfs.working.dir=kylin.env.hdfs-working-dir
+
+
+### METADATA ###
+
+kylin.metadata.url=kylin.metadata.url
+kylin.realization.providers=kylin.metadata.realization-providers
+kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
+kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types.
+
+
+### Dictionary ###
+
+kylin.table.snapshot.max_mb=kylin.snapshot.max-mb
+kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry
+kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb
+kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry
+kylin.dict.growing.enabled=kylin.dictionary.growing-enabled
+kylin.dict.append.entry.size=kylin.dictionary.append-entry-size
+kylin.dict.append.cache.size=kylin.dictionary.append-cache-size
+
+
+### CUBE ###
+
+kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio
+kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio
+kylin.cube.algorithm=kylin.cube.algorithm
+kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold
+kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit
+kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size
+kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination
+kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid
+kylin.cube.building.segment.max=kylin.cube.max-building-segments
+
+
+### JOB ###
+
+kylin.job.log.dir=kylin.job.log-dir
+kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
+kylin.job.allow.empty.segment=kylin.job.allow-empty-segment
+kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs
+kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage
+kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision
+kylin.job.dependency.filterlist=kylin.job.dependency-filter-list
+kylin.job.retry=kylin.job.retry
+kylin.job.controller.lock=kylin.job.lock
+kylin.scheduler.=kylin.job.scheduler.provider.
+kylin.enable.scheduler=kylin.job.scheduler.default
+
+mail.enabled=kylin.job.notification-enabled
+mail.host=kylin.job.notification-mail-host
+mail.username=kylin.job.notification-mail-username
+mail.password=kylin.job.notification-mail-password
+mail.sender=kylin.job.notification-mail-sender
+kylin.job.admin.dls=kylin.job.notification-admin-emails
+
+#for dev
+kylin.job.run.as.remote.cmd=kylin.job.use-remote-cli
+kylin.job.remote.cli.port=kylin.job.remote-cli-port
+kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname
+kylin.job.remote.cli.username=kylin.job.remote-cli-username
+kylin.job.remote.cli.password=kylin.job.remote-cli-password
+
+
+### ENGINE ###
+
+kylin.cube.engine.=kylin.engine.provider.
+kylin.default.cube.engine=kylin.engine.default
+
+kylin.job.mr.lib.dir=kylin.engine.mr.lib-dir
+kylin.job.mr.config.override.=kylin.engine.mr.config-override.
+kylin.job.jar=kylin.engine.mr.job-jar
+kylin.job.mapreduce.default.reduce.input.mb=kylin.engine.mr.reduce-input-mb
+kylin.job.mapreduce.default.reduce.count.ratio=kylin.engine.mr.reduce-count-ratio
+kylin.job.mapreduce.min.reducer.number=kylin.engine.mr.min-reducer-number
+kylin.job.mapreduce.max.reducer.number=kylin.engine.mr.max-reducer-number
+kylin.job.mapreduce.mapper.input.rows=kylin.engine.mr.mapper-input-rows
+kylin.job.uhc.reducer.count=kylin.engine.mr.uhc-reducer-count
+kylin.job.yarn.app.rest.check.interval.seconds=kylin.engine.mr.yarn-check-interval-seconds
+
+kylin.job.jar.spark=kylin.engine.spark.job-jar
+kylin.spark.home=kylin.engine.spark.spark-home
+kylin.spark.master=kylin.engine.spark.spark-master
+
+#deprecated
+kylin.job.yarn.app.rest.check.status.url=kylin.engine.mr.yarn-check-status-url
+
+
+### SOURCE ###
+
+kylin.source.engine.=kylin.source.provider.
+kylin.hive.config.override.=kylin.source.hive.config-override.
+kylin.hive.keep.flat.table=kylin.source.hive.keep-flat-table
+kylin.job.hive.database.for.intermediatetable=kylin.source.hive.database-for-flat-table
+kylin.job.hive.intermediatetable.redistribute.enabled=kylin.source.hive.redistribute-flat-table
+kylin.hive.client=kylin.source.hive.client
+kylin.hive.beeline.params=kylin.source.hive.beeline-params
+kylin.hive.create.flat.table.method=kylin.source.hive.create-flat-table-method
+
+#deprecated
+hive.url=kylin.source.hive.url
+hive.user=kylin.source.hive.user
+hive.password=kylin.source.hive.password
+hive.table.location.=kylin.source.hive.table-location.
+
+
+### STORAGE ###
+
+kylin.storage.url=kylin.storage.url
+kylin.storage.engine.=kylin.storage.provider.
+kylin.default.storage.engine=kylin.storage.default
+
+kylin.hbase.cluster.fs=kylin.storage.hbase.cluster-fs
+kylin.hbase.cluster.hdfs.config.file=kylin.storage.hbase.cluster-hdfs-config-file
+kylin.coprocessor.local.jar=kylin.storage.hbase.coprocessor-local-jar
+kylin.hbase.region.count.min=kylin.storage.hbase.min-region-count
+kylin.hbase.region.count.max=kylin.storage.hbase.max-region-count
+kylin.hbase.hfile.size.gb=kylin.storage.hbase.hfile-size-gb
+kylin.query.run.local.coprocessor=kylin.storage.hbase.run-local-coprocessor
+kylin.query.coprocessor.mem.gb=kylin.storage.hbase.coprocessor-mem-gb
+kylin.query.coprocessor.timeout.seconds=kylin.storage.hbase.coprocessor-timeout-seconds
+kylin.query.scan.fuzzykey.max=kylin.storage.hbase.max-fuzzykey-scan
+kylin.query.storage.visit.scanrange.max=kylin.storage.hbase.max-visit-scanrange
+kylin.query.storage.default.gtstorage=kylin.storage.hbase.gtstorage
+kylin.hbase.scan.cache_rows=kylin.storage.hbase.scan-cache-rows
+kylin.hbase.region.cut=kylin.storage.hbase.region-cut-gb
+kylin.hbase.scan.max_result_size=kylin.storage.hbase.max-scan-result-bytes
+kylin.hbase.default.compression.codec=kylin.storage.hbase.compression-codec
+kylin.hbase.default.encoding=kylin.storage.hbase.rowkey-encoding
+kylin.hbase.default.block.size=kylin.storage.hbase.block-size-bytes
+kylin.hbase.small.family.block.size=kylin.storage.hbase.small-family-block-size-bytes
+kylin.owner=kylin.storage.hbase.owner-tag
+kylin.query.endpoint.compression.result=kylin.storage.hbase.endpoint-compress-result
+kylin.query.hbase.hconnection.threads.max=kylin.storage.hbase.max-hconnection-threads
+kylin.query.hbase.hconnection.threads.core=kylin.storage.hbase.core-hconnection-threads
+kylin.query.hbase.hconnection.threads.alive.seconds=kylin.storage.hbase.hconnection-threads-alive-seconds
+
+
+### QUERY ###
+
+kylin.query.pushdown.limit.max=kylin.query.max-limit-pushdown
+kylin.query.scan.threshold=kylin.query.scan-threshold
+kylin.query.filter.derived_in.max=kylin.query.derived-filter-translation-threshold
+kylin.query.badquery.stacktrace.depth=kylin.query.badquery-stacktrace-depth
+kylin.query.badquery.history.num=kylin.query.badquery-history-number
+kylin.query.badquery.alerting.seconds=kylin.query.badquery-alerting-seconds
+kylin.query.badquery.detect.interval.seconds=kylin.query.badquery-detect-interval
+kylin.query.badquery.persistent.enable=kylin.query.badquery-persistent-enabled
+kylin.query.transformers=kylin.query.transformers
+kylin.query.cache.enabled=kylin.query.cache-enabled
+kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration
+kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count
+kylin.query.mem.budget=kylin.query.memory-budget
+kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function
+kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct
+kylin.query.security.enabled=kylin.query.security-enabled
+kylin.query.access.controller=kylin.query.access-controller
+kylin.query.udf.=kylin.query.udf.
+
+
+### SERVER ###
+
+kylin.init.tasks=kylin.server.init-tasks
+kylin.server.mode=kylin.server.mode
+kylin.cluster.name=kylin.server.cluster-name
+kylin.rest.servers=kylin.server.cluster-servers
+kylin.rest.workers.per.server=kylin.server.sequence-sql.workers-per-server
+kylin.query.sequence.expire.time=kylin.server.sequence-sql.expire-time
+kylin.query.metrics.enabled=kylin.server.query-metrics-enabled
+kylin.query.metrics.percentiles.intervals=kylin.server.query-metrics-percentiles-intervals
+
+
+### WEB ###
+
+kylin.rest.timezone=kylin.web.timezone
+crossdomain.enable=kylin.web.cross-domain-enabled
### TEST ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 9257718..886d8ee 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -56,7 +56,7 @@ public class KylinConfigTest extends LocalFileMetadataTestCase {
@Test
public void testBackwardCompatibility() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- final String oldk = "kylin.test.bcc.old.key";
+ final String oldk = "kylin.test.bcc.new.key";
final String newk = "kylin.test.bcc.new.key";
assertNull(config.getOptional(oldk));
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java b/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
index df06221..f0d5e4f 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
@@ -51,7 +51,7 @@ public class MailServiceTest extends LocalFileMetadataTestCase {
boolean sent = sendTestEmail(mailservice);
assert sent;
- // set mail.enabled=false, and run again, this time should be no mail delivered
+ // set kylin.job.notification-enabled=false, and run again, this time should be no mail delivered
config.setMailEnabled(false);
mailservice = new MailService(config);
sent = sendTestEmail(mailservice);
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java b/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java
index 4ddb06a..74012a4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java
@@ -1,130 +1,130 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.common;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-public class FuzzyValueCombination {
-
- private static class Dim<K, V> {
- K col;
- Set<V> values;
- }
-
- private static final Set SINGLE_NULL_SET = Sets.newHashSet();
-
- static {
- SINGLE_NULL_SET.add(null);
- }
-
- public static <K, V> List<Map<K, V>> calculate(Map<K, Set<V>> fuzzyValues, long cap) {
- Collections.emptyMap();
- Dim<K, V>[] dims = toDims(fuzzyValues);
- // If a query has many IN clause and each IN clause has many values, then it will easily generate
- // thousands of fuzzy keys. When there are lots of fuzzy keys, the scan performance is bottle necked
- // on it. So simply choose to abandon all fuzzy keys in this case.
- if (exceedCap(dims, cap)) {
- return Lists.newArrayList();
- } else {
- return combination(dims);
- }
- }
-
- @SuppressWarnings("unchecked")
- private static <K, V> List<Map<K, V>> combination(Dim<K, V>[] dims) {
-
- List<Map<K, V>> result = Lists.newArrayList();
-
- int emptyDims = 0;
- for (Dim dim : dims) {
- if (dim.values.isEmpty()) {
- dim.values = SINGLE_NULL_SET;
- emptyDims++;
- }
- }
- if (emptyDims == dims.length) {
- return result;
- }
-
- Map<K, V> r = Maps.newHashMap();
- Iterator<V>[] iters = new Iterator[dims.length];
- int level = 0;
- while (true) {
- Dim<K, V> dim = dims[level];
- if (iters[level] == null) {
- iters[level] = dim.values.iterator();
- }
-
- Iterator<V> it = iters[level];
- if (it.hasNext() == false) {
- if (level == 0)
- break;
- r.remove(dim.col);
- iters[level] = null;
- level--;
- continue;
- }
-
- r.put(dim.col, it.next());
- if (level == dims.length - 1) {
- result.add(new HashMap<K, V>(r));
- } else {
- level++;
- }
- }
- return result;
- }
-
- private static <K, V> Dim<K, V>[] toDims(Map<K, Set<V>> fuzzyValues) {
- Dim[] dims = new Dim[fuzzyValues.size()];
- int i = 0;
- for (Entry<K, Set<V>> entry : fuzzyValues.entrySet()) {
- dims[i] = new Dim<K, V>();
- dims[i].col = entry.getKey();
- dims[i].values = entry.getValue();
- if (dims[i].values == null)
- dims[i].values = Collections.emptySet();
- i++;
- }
- return dims;
- }
-
- private static boolean exceedCap(Dim[] dims, long cap) {
- return combCount(dims) > cap;
- }
-
- private static long combCount(Dim[] dims) {
- long count = 1;
- for (Dim dim : dims) {
- count *= Math.max(dim.values.size(), 1);
- }
- return count;
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.common;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class FuzzyValueCombination {
+
+ private static class Dim<K, V> {
+ K col;
+ Set<V> values;
+ }
+
+ private static final Set SINGLE_NULL_SET = Sets.newHashSet();
+
+ static {
+ SINGLE_NULL_SET.add(null);
+ }
+
+ public static <K, V> List<Map<K, V>> calculate(Map<K, Set<V>> fuzzyValues, long cap) {
+ Collections.emptyMap();
+ Dim<K, V>[] dims = toDims(fuzzyValues);
+ // If a query has many IN clause and each IN clause has many values, then it will easily generate
+ // thousands of fuzzy keys. When there are lots of fuzzy keys, the scan performance is bottle necked
+ // on it. So simply choose to abandon all fuzzy keys in this case.
+ if (exceedCap(dims, cap)) {
+ return Lists.newArrayList();
+ } else {
+ return combination(dims);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <K, V> List<Map<K, V>> combination(Dim<K, V>[] dims) {
+
+ List<Map<K, V>> result = Lists.newArrayList();
+
+ int emptyDims = 0;
+ for (Dim dim : dims) {
+ if (dim.values.isEmpty()) {
+ dim.values = SINGLE_NULL_SET;
+ emptyDims++;
+ }
+ }
+ if (emptyDims == dims.length) {
+ return result;
+ }
+
+ Map<K, V> r = Maps.newHashMap();
+ Iterator<V>[] iters = new Iterator[dims.length];
+ int level = 0;
+ while (true) {
+ Dim<K, V> dim = dims[level];
+ if (iters[level] == null) {
+ iters[level] = dim.values.iterator();
+ }
+
+ Iterator<V> it = iters[level];
+ if (it.hasNext() == false) {
+ if (level == 0)
+ break;
+ r.remove(dim.col);
+ iters[level] = null;
+ level--;
+ continue;
+ }
+
+ r.put(dim.col, it.next());
+ if (level == dims.length - 1) {
+ result.add(new HashMap<K, V>(r));
+ } else {
+ level++;
+ }
+ }
+ return result;
+ }
+
+ private static <K, V> Dim<K, V>[] toDims(Map<K, Set<V>> fuzzyValues) {
+ Dim[] dims = new Dim[fuzzyValues.size()];
+ int i = 0;
+ for (Entry<K, Set<V>> entry : fuzzyValues.entrySet()) {
+ dims[i] = new Dim<K, V>();
+ dims[i].col = entry.getKey();
+ dims[i].values = entry.getValue();
+ if (dims[i].values == null)
+ dims[i].values = Collections.emptySet();
+ i++;
+ }
+ return dims;
+ }
+
+ private static boolean exceedCap(Dim[] dims, long cap) {
+ return combCount(dims) > cap;
+ }
+
+ private static long combCount(Dim[] dims) {
+ long count = 1;
+ for (Dim dim : dims) {
+ count *= Math.max(dim.values.size(), 1);
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
index d5918aa..05efb5e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
@@ -1,143 +1,143 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.cuboid;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.model.CubeDesc;
-
-/**
- * @author yangli9
- */
-public class CuboidCLI {
-
- public static void main(String[] args) throws IOException {
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- if ("test".equals(args[0])) {
- CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
- simulateCuboidGeneration(cubeDesc, true);
- }
- }
-
- public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
- CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
- long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
- Collection<Long> cuboidSet = new TreeSet<Long>();
- cuboidSet.add(baseCuboid);
- LinkedList<Long> cuboidQueue = new LinkedList<Long>();
- cuboidQueue.push(baseCuboid);
- while (!cuboidQueue.isEmpty()) {
- long cuboid = cuboidQueue.pop();
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
- for (Long sc : spnanningCuboids) {
- boolean notfound = cuboidSet.add(sc);
- if (!notfound) {
- throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
- }
-
- cuboidQueue.push(sc);
-
- }
- }
-
- if (validate) {
- //only run this for test purpose, performance is bad when # of dims is large
- TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc);
- System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()])));
- if (enumCuboids.equals(cuboidSet) == false) {
- throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
- }
-
- //check all valid and invalid
- for (long i = 0; i < baseCuboid; ++i) {
- if (cuboidSet.contains(i)) {
- if (!Cuboid.isValid(cubeDesc, i)) {
- throw new RuntimeException();
- }
-
- if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
- throw new RuntimeException();
- }
- } else {
- if (Cuboid.isValid(cubeDesc, i)) {
- throw new RuntimeException();
- }
-
- long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
- if (corrected == i) {
- throw new RuntimeException();
- }
-
- if (!Cuboid.isValid(cubeDesc, corrected)) {
- throw new RuntimeException();
- }
-
- if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
- throw new RuntimeException();
- }
- }
- }
- }
-
- return cuboidSet.size();
-
- }
-
- public static TreeSet<Long> enumCalcCuboidCount(CubeDesc cube) {
- long baseCuboid = Cuboid.getBaseCuboidId(cube);
- TreeSet<Long> expectedCuboids = new TreeSet<Long>();
- for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
- if (Cuboid.isValid(cube, cuboid)) {
- expectedCuboids.add(cuboid);
- }
- }
- return expectedCuboids;
- }
-
- public static int[] calculateAllLevelCount(CubeDesc cube) {
- int levels = cube.getBuildLevel();
- int[] allLevelCounts = new int[levels + 1];
-
- CuboidScheduler scheduler = new CuboidScheduler(cube);
- LinkedList<Long> nextQueue = new LinkedList<Long>();
- LinkedList<Long> currentQueue = new LinkedList<Long>();
- long baseCuboid = Cuboid.getBaseCuboidId(cube);
- currentQueue.push(baseCuboid);
-
- for (int i = 0; i <= levels; i++) {
- allLevelCounts[i] = currentQueue.size();
- while (!currentQueue.isEmpty()) {
- long cuboid = currentQueue.pop();
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
- nextQueue.addAll(spnanningCuboids);
- }
- currentQueue = nextQueue;
- nextQueue = new LinkedList<Long>();
- }
-
- return allLevelCounts;
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.TreeSet;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+
+/**
+ * @author yangli9
+ */
+public class CuboidCLI {
+
+ public static void main(String[] args) throws IOException {
+ CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+ if ("test".equals(args[0])) {
+ CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
+ simulateCuboidGeneration(cubeDesc, true);
+ }
+ }
+
+ public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
+ CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
+ long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
+ Collection<Long> cuboidSet = new TreeSet<Long>();
+ cuboidSet.add(baseCuboid);
+ LinkedList<Long> cuboidQueue = new LinkedList<Long>();
+ cuboidQueue.push(baseCuboid);
+ while (!cuboidQueue.isEmpty()) {
+ long cuboid = cuboidQueue.pop();
+ Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
+ for (Long sc : spnanningCuboids) {
+ boolean notfound = cuboidSet.add(sc);
+ if (!notfound) {
+ throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
+ }
+
+ cuboidQueue.push(sc);
+
+ }
+ }
+
+ if (validate) {
+ //only run this for test purpose, performance is bad when # of dims is large
+ TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc);
+ System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()])));
+ if (enumCuboids.equals(cuboidSet) == false) {
+ throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
+ }
+
+ //check all valid and invalid
+ for (long i = 0; i < baseCuboid; ++i) {
+ if (cuboidSet.contains(i)) {
+ if (!Cuboid.isValid(cubeDesc, i)) {
+ throw new RuntimeException();
+ }
+
+ if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) {
+ throw new RuntimeException();
+ }
+ } else {
+ if (Cuboid.isValid(cubeDesc, i)) {
+ throw new RuntimeException();
+ }
+
+ long corrected = Cuboid.translateToValidCuboid(cubeDesc, i);
+ if (corrected == i) {
+ throw new RuntimeException();
+ }
+
+ if (!Cuboid.isValid(cubeDesc, corrected)) {
+ throw new RuntimeException();
+ }
+
+ if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) {
+ throw new RuntimeException();
+ }
+ }
+ }
+ }
+
+ return cuboidSet.size();
+
+ }
+
+ public static TreeSet<Long> enumCalcCuboidCount(CubeDesc cube) {
+ long baseCuboid = Cuboid.getBaseCuboidId(cube);
+ TreeSet<Long> expectedCuboids = new TreeSet<Long>();
+ for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
+ if (Cuboid.isValid(cube, cuboid)) {
+ expectedCuboids.add(cuboid);
+ }
+ }
+ return expectedCuboids;
+ }
+
+ public static int[] calculateAllLevelCount(CubeDesc cube) {
+ int levels = cube.getBuildLevel();
+ int[] allLevelCounts = new int[levels + 1];
+
+ CuboidScheduler scheduler = new CuboidScheduler(cube);
+ LinkedList<Long> nextQueue = new LinkedList<Long>();
+ LinkedList<Long> currentQueue = new LinkedList<Long>();
+ long baseCuboid = Cuboid.getBaseCuboidId(cube);
+ currentQueue.push(baseCuboid);
+
+ for (int i = 0; i <= levels; i++) {
+ allLevelCounts[i] = currentQueue.size();
+ while (!currentQueue.isEmpty()) {
+ long cuboid = currentQueue.pop();
+ Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
+ nextQueue.addAll(spnanningCuboids);
+ }
+ currentQueue = nextQueue;
+ nextQueue = new LinkedList<Long>();
+ }
+
+ return allLevelCounts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
index ec0d39d..99dacc8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
@@ -1,47 +1,47 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.kv;
-
-public class RowConstants {
-
- public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
-
- // row key lower bound
- public static final byte ROWKEY_LOWER_BYTE = 0;
- // row key upper bound
- public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff;
-
- // row key cuboid id length
- public static final int ROWKEY_CUBOIDID_LEN = 8;
- // row key shard length
- public static final int ROWKEY_SHARDID_LEN = 2;
-
- public static final int ROWKEY_SHARD_AND_CUBOID_LEN = ROWKEY_CUBOIDID_LEN + ROWKEY_SHARDID_LEN;
-
- public static final byte BYTE_ZERO = 0;
- public static final byte BYTE_ONE = 1;
-
- // row value delimiter
- public static final byte ROWVALUE_DELIMITER_BYTE = 7;
- public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7);
- public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 };
-
- public static final int ROWKEY_BUFFER_SIZE = 65 * 256;// a little more than 64 dimensions * 256 bytes each
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.kv;
+
+public class RowConstants {
+
+ public static final int ROWKEY_COL_DEFAULT_LENGTH = 256;
+
+ // row key lower bound
+ public static final byte ROWKEY_LOWER_BYTE = 0;
+ // row key upper bound
+ public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff;
+
+ // row key cuboid id length
+ public static final int ROWKEY_CUBOIDID_LEN = 8;
+ // row key shard length
+ public static final int ROWKEY_SHARDID_LEN = 2;
+
+ public static final int ROWKEY_SHARD_AND_CUBOID_LEN = ROWKEY_CUBOIDID_LEN + ROWKEY_SHARDID_LEN;
+
+ public static final byte BYTE_ZERO = 0;
+ public static final byte BYTE_ONE = 1;
+
+ // row value delimiter
+ public static final byte ROWVALUE_DELIMITER_BYTE = 7;
+ public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7);
+ public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 };
+
+ public static final int ROWKEY_BUFFER_SIZE = 65 * 256;// a little more than 64 dimensions * 256 bytes each
+
+}