You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:46:41 UTC
[20/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/com/kylinolap/job/hadoop/cardinality/ColumnCardinalityReducer.java
new file mode 100644
index 0000000..06c98ee
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cardinality/ColumnCardinalityReducer.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cardinality;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import com.kylinolap.common.hll.HyperLogLogPlusCounter;
+import com.kylinolap.cube.kv.RowConstants;
+
+/**
+ * @author Jack
+ *
+ */
+public class ColumnCardinalityReducer extends Reducer<IntWritable, BytesWritable, IntWritable, LongWritable> {
+
+ public static final int ONE = 1;
+ private Map<Integer, HyperLogLogPlusCounter> hllcMap = new HashMap<Integer, HyperLogLogPlusCounter>();
+
+ @Override
+ public void reduce(IntWritable key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
+ for (BytesWritable v : values) {
+ int skey = key.get();
+ ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
+ HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
+ hll.readRegisters(buffer);
+ getHllc(skey).merge(hll);
+ hll.clear();
+ }
+ }
+
+ private HyperLogLogPlusCounter getHllc(Integer key) {
+ if (!hllcMap.containsKey(key)) {
+ hllcMap.put(key, new HyperLogLogPlusCounter());
+ }
+ return hllcMap.get(key);
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ List<Integer> keys = new ArrayList<Integer>();
+ Iterator<Integer> it = hllcMap.keySet().iterator();
+ while (it.hasNext()) {
+ keys.add(it.next());
+ }
+ Collections.sort(keys);
+ it = keys.iterator();
+ while (it.hasNext()) {
+ int key = it.next();
+ HyperLogLogPlusCounter hllc = hllcMap.get(key);
+ ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
+ // context.write(new Text("ErrorRate_" + key), new
+ // LongWritable((long)hllc.getErrorRate()));
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cardinality/HiveColumnCardinalityJob.java
new file mode 100644
index 0000000..b6ea002
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cardinality/HiveColumnCardinalityJob.java
@@ -0,0 +1,254 @@
+package com.kylinolap.job.hadoop.cardinality;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+
+public class HiveColumnCardinalityJob extends AbstractHadoopJob {
+ public static final String JOB_TITLE = "Kylin Hive Column Cardinality Job";
+
+ @SuppressWarnings("static-access")
+ protected static final Option OPTION_FORMAT = OptionBuilder.withArgName("input format").hasArg().isRequired(true).withDescription("The file format").create("iformat");
+
+ @SuppressWarnings("static-access")
+ protected static final Option OPTION_INPUT_DELIM = OptionBuilder.withArgName("input_dilim").hasArg().isRequired(false).withDescription("Input delim").create("idelim");
+
+ public static final String KEY_INPUT_DELIM = "INPUT_DELIM";
+ public static final String OUTPUT_PATH = "/tmp/cardinality";
+
+ /**
+ * This is the jar path
+ */
+ private String jarPath;
+ private Configuration conf;
+
+ /**
+ * MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY
+ */
+ private String tokenPath;
+
+ public HiveColumnCardinalityJob() {
+
+ }
+
+ public HiveColumnCardinalityJob(String path, String tokenPath) {
+ this.jarPath = path;
+ this.tokenPath = tokenPath;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.conf.Configured#getConf()
+ */
+ @Override
+ public Configuration getConf() {
+ if (conf != null) {
+ return conf;
+ }
+ conf = new JobConf();
+ String path = "/apache/hadoop/conf/";
+ File file = new File(path);
+ if (file.isDirectory()) {
+ File[] files = file.listFiles();
+ for (int i = 0; i < files.length; i++) {
+ File tmp = files[i];
+ if (tmp.getName().endsWith(".xml")) {
+ try {
+ conf.addResource(new FileInputStream(tmp));
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ // conf.addResource("/apache/hadoop/conf/mapred-site.xml");
+ if (tokenPath != null) {
+ conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, tokenPath);
+ conf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ try {
+ UserGroupInformation.loginUserFromKeytab("b_kylin@CORP.EBAY.COM", "~/.keytabs/b_kylin.keytab");
+ System.out.println("###" + UserGroupInformation.getLoginUser());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return conf;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int run(String[] args) throws Exception {
+
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ options.addOption(OPTION_FORMAT);
+ options.addOption(OPTION_INPUT_DELIM);
+
+ parseOptions(options, args);
+
+ // start job
+ String jobName = JOB_TITLE + getOptionsAsString();
+ System.out.println("Starting: " + jobName);
+ Configuration conf = getConf();
+ job = Job.getInstance(conf, jobName);
+
+ // set job configuration - basic
+ if (jarPath == null || !new File(jarPath).exists()) {
+ job.setJarByClass(getClass());
+ } else {
+ job.setJar(jarPath);
+ }
+ FileInputFormat.setInputDirRecursive(job, true);
+ addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
+
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+ FileOutputFormat.setOutputPath(job, output);
+ job.getConfiguration().set("dfs.block.size", "67108864");
+
+ String format = getOptionValue(OPTION_FORMAT);
+ @SuppressWarnings("rawtypes")
+ Class cformat = getFormat(format);
+ String delim = getOptionValue(OPTION_INPUT_DELIM);
+ if (delim != null) {
+ if (delim.equals("t")) {
+ delim = "\t";
+ } else if (delim.equals("177")) {
+ delim = "\177";
+ }
+ job.getConfiguration().set(KEY_INPUT_DELIM, delim);
+ }
+
+ // Mapper
+ job.setInputFormatClass(cformat);
+ job.setMapperClass(ColumnCardinalityMapper.class);
+ job.setMapOutputKeyClass(IntWritable.class);
+ job.setMapOutputValueClass(BytesWritable.class);
+
+ // Reducer - only one
+ job.setReducerClass(ColumnCardinalityReducer.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ job.setOutputKeyClass(IntWritable.class);
+ job.setOutputValueClass(LongWritable.class);
+ job.setNumReduceTasks(1);
+
+ this.deletePath(job.getConfiguration(), output);
+
+ int result = waitForCompletion(job);
+ return result;
+ } catch (Exception e) {
+ printUsage(options);
+ e.printStackTrace(System.err);
+ log.error(e.getLocalizedMessage(), e);
+ return 2;
+ }
+
+ }
+
+ /**
+ * @param format
+ * @throws ClassNotFoundException
+ */
+ @SuppressWarnings("rawtypes")
+ private Class getFormat(String format) throws ClassNotFoundException {
+ if (format.endsWith(".TextInputFormat")) {
+ return Class.forName("org.apache.hadoop.mapreduce.lib.input.TextInputFormat");
+ } else if (format.endsWith(".SequenceFileInputFormat")) {
+ return Class.forName("org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat");
+ } else {
+ return Class.forName(format);
+ }
+
+ }
+
+ public static void main1(String[] args) throws Exception {
+ int exitCode = ToolRunner.run(new HiveColumnCardinalityJob(), args);
+ System.exit(exitCode);
+ }
+
+ public static void main(String[] args) {
+
+ String location = "hdfs://apollo-phx-nn.vip.ebay.com:8020/tmp/f1a98d8a-26b9-452e-ab7b-9f01e5a6459b/shipping_sisense_cube_desc_intermediate_table";
+ String tempName = "test";
+ String inputFormat = "org.apache.hadoop.mapred.SequenceFileInputFormat";
+ String delim = "177";
+ String jarPath = "/usr/lib/kylin/kylin-index-latest.jar";
+
+ args = new String[] { "-input", location, "-output", "/tmp/cardinality/" + tempName, "-iformat", inputFormat, "-idelim", delim };
+ HiveColumnCardinalityJob job = new HiveColumnCardinalityJob(jarPath, "/tmp/krb5cc_882");
+ try {
+ ToolRunner.run(job, args);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public List<String> readLines(Path location, Configuration conf) throws Exception {
+ FileSystem fileSystem = FileSystem.get(location.toUri(), conf);
+ CompressionCodecFactory factory = new CompressionCodecFactory(conf);
+ FileStatus[] items = fileSystem.listStatus(location);
+ if (items == null)
+ return new ArrayList<String>();
+ List<String> results = new ArrayList<String>();
+ for (FileStatus item : items) {
+
+ // ignoring files like _SUCCESS
+ if (item.getPath().getName().startsWith("_")) {
+ continue;
+ }
+
+ CompressionCodec codec = factory.getCodec(item.getPath());
+ InputStream stream = null;
+
+ // check if we have a compression codec we need to use
+ if (codec != null) {
+ stream = codec.createInputStream(fileSystem.open(item.getPath()));
+ } else {
+ stream = fileSystem.open(item.getPath());
+ }
+
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(stream, writer, "UTF-8");
+ String raw = writer.toString();
+ for (String str : raw.split("\n")) {
+ results.add(str);
+ }
+ }
+ return results;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidJob.java
new file mode 100644
index 0000000..cc25b13
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidJob.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * @author honma
+ *
+ */
+
+public class BaseCuboidJob extends CuboidJob {
+ public BaseCuboidJob() {
+ this.setMapperClass(BaseCuboidMapper.class);
+ }
+
+ public static void main(String[] args) throws Exception {
+ CuboidJob job = new BaseCuboidJob();
+ int exitCode = ToolRunner.run(job, args);
+ System.exit(exitCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapper.java
new file mode 100644
index 0000000..39b3918
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapper.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.CubeSegment;
+import com.kylinolap.cube.CubeSegmentStatusEnum;
+import com.kylinolap.cube.common.BytesSplitter;
+import com.kylinolap.cube.common.SplittedBytes;
+import com.kylinolap.cube.cuboid.Cuboid;
+import com.kylinolap.cube.kv.AbstractRowKeyEncoder;
+import com.kylinolap.cube.kv.RowConstants;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.job.hadoop.hive.JoinedFlatTableDesc;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.FunctionDesc;
+import com.kylinolap.metadata.model.cube.MeasureDesc;
+import com.kylinolap.metadata.model.cube.ParameterDesc;
+
+/**
+ * @author George Song (ysong1)
+ */
+public class BaseCuboidMapper<KEYIN> extends Mapper<KEYIN, Text, Text, Text> {
+
+ private static final Logger logger = LoggerFactory.getLogger(BaseCuboidMapper.class);
+
+ public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
+ public static final byte[] ONE = Bytes.toBytes("1");
+
+ private String cubeName;
+ private String segmentName;
+ private Cuboid baseCuboid;
+ private CubeInstance cube;
+ private CubeDesc cubeDesc;
+ private CubeSegment cubeSegment;
+ private List<byte[]> nullBytes;
+
+ private JoinedFlatTableDesc intermediateTableDesc;
+ private String intermediateTableRowDelimiter;
+ private byte byteRowDelimiter;
+
+ private int counter;
+ private int errorRecordCounter;
+ private Text outputKey = new Text();
+ private Text outputValue = new Text();
+ private Object[] measures;
+ private byte[][] keyBytesBuf;
+ private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+
+ private BytesSplitter bytesSplitter;
+ private AbstractRowKeyEncoder rowKeyEncoder;
+ private MeasureCodec measureCodec;
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+ segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
+ intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
+ if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
+ throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
+ }
+
+ byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];
+
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
+
+ cube = CubeManager.getInstance(config).getCube(cubeName);
+ cubeDesc = cube.getDescriptor();
+ cubeSegment = cube.getSegment(segmentName, CubeSegmentStatusEnum.NEW);
+
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+
+ intermediateTableDesc = new JoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
+
+ bytesSplitter = new BytesSplitter(200, 4096);
+ rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
+
+ measureCodec = new MeasureCodec(cubeDesc.getMeasures());
+ measures = new Object[cubeDesc.getMeasures().size()];
+
+ int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
+ keyBytesBuf = new byte[colCount][];
+
+ initNullBytes();
+ }
+
+ private void initNullBytes() {
+ nullBytes = Lists.newArrayList();
+ nullBytes.add(HIVE_NULL);
+ String[] nullStrings = cubeDesc.getNullStrings();
+ if (nullStrings != null) {
+ for (String s : nullStrings) {
+ nullBytes.add(Bytes.toBytes(s));
+ }
+ }
+ }
+
+ private boolean isNull(byte[] v) {
+ for (byte[] nullByte : nullBytes) {
+ if (Bytes.equals(v, nullByte))
+ return true;
+ }
+ return false;
+ }
+
+ private byte[] buildKey(SplittedBytes[] splitBuffers) {
+ int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
+ for (int i = 0; i < baseCuboid.getColumns().size(); i++) {
+ int index = rowKeyColumnIndexes[i];
+ keyBytesBuf[i] = Arrays.copyOf(splitBuffers[index].value, splitBuffers[index].length);
+ if (isNull(keyBytesBuf[i])) {
+ keyBytesBuf[i] = null;
+ }
+ }
+ return rowKeyEncoder.encode(keyBytesBuf);
+ }
+
+ private void buildValue(SplittedBytes[] splitBuffers) {
+
+ for (int i = 0; i < measures.length; i++) {
+ byte[] valueBytes = getValueBytes(splitBuffers, i);
+ measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes);
+ }
+
+ valueBuf.clear();
+ measureCodec.encode(measures, valueBuf);
+ }
+
+ private byte[] getValueBytes(SplittedBytes[] splitBuffers, int measureIdx) {
+ MeasureDesc desc = cubeDesc.getMeasures().get(measureIdx);
+ FunctionDesc func = desc.getFunction();
+ ParameterDesc paramDesc = func.getParameter();
+ int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
+
+ byte[] result = null;
+
+ // constant
+ if (flatTableIdx == null) {
+ result = Bytes.toBytes(paramDesc.getValue());
+ }
+ // column values
+ else {
+ // for multiple columns, their values are joined
+ for (int i = 0; i < flatTableIdx.length; i++) {
+ SplittedBytes split = splitBuffers[flatTableIdx[i]];
+ if (result == null) {
+ result = Arrays.copyOf(split.value, split.length);
+ } else {
+ byte[] newResult = new byte[result.length + split.length];
+ System.arraycopy(result, 0, newResult, 0, result.length);
+ System.arraycopy(split.value, 0, newResult, result.length, split.length);
+ result = newResult;
+ }
+ }
+ }
+
+ if (func.isCount() || func.isHolisticCountDistinct()) {
+ // note for holistic count distinct, this value will be ignored
+ result = ONE;
+ }
+
+ if (isNull(result)) {
+ result = null;
+ }
+
+ return result;
+ }
+
+ @Override
+ public void map(KEYIN key, Text value, Context context) throws IOException, InterruptedException {
+ counter++;
+ if (counter % BatchConstants.COUNTER_MAX == 0) {
+ logger.info("Handled " + counter + " records!");
+ }
+
+ bytesSplitter.split(value.getBytes(), value.getLength(), byteRowDelimiter);
+
+ try {
+ byte[] rowKey = buildKey(bytesSplitter.getSplitBuffers());
+ outputKey.set(rowKey, 0, rowKey.length);
+
+ buildValue(bytesSplitter.getSplitBuffers());
+ outputValue.set(valueBuf.array(), 0, valueBuf.position());
+
+ context.write(outputKey, outputValue);
+ } catch (Exception ex) {
+ handleErrorRecord(bytesSplitter.getSplitBuffers(), ex);
+ }
+ }
+
+ private void handleErrorRecord(SplittedBytes[] splitBuffers, Exception ex) throws IOException {
+
+ StringBuilder buf = new StringBuilder();
+ buf.append("Error record: [");
+ for (int i = 0; i < splitBuffers.length; i++) {
+ if (i > 0)
+ buf.append(", ");
+
+ buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length));
+ }
+ buf.append("] -- ");
+ buf.append(ex.toString());
+ System.err.println(buf.toString());
+ ex.printStackTrace(System.err);
+
+ errorRecordCounter++;
+ if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
+ if (ex instanceof IOException)
+ throw (IOException) ex;
+ else if (ex instanceof RuntimeException)
+ throw (RuntimeException) ex;
+ else
+ throw new RuntimeException("", ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileJob.java
new file mode 100644
index 0000000..1f939d1
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileJob.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.File;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
+import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+
+public class CubeHFileJob extends AbstractHadoopJob {
+
+ protected static final Logger log = LoggerFactory.getLogger(CubeHFileJob.class);
+
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_JOB_NAME);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ options.addOption(OPTION_HTABLE_NAME);
+ parseOptions(options, args);
+
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+ String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+
+ CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+ job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+
+ File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
+ if (JarFile.exists()) {
+ job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
+ } else {
+ job.setJarByClass(this.getClass());
+ }
+
+ addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
+ FileOutputFormat.setOutputPath(job, output);
+
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setMapperClass(CubeHFileMapper.class);
+ job.setReducerClass(KeyValueSortReducer.class);
+
+ // set job configuration
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ Configuration conf = HBaseConfiguration.create(getConf());
+ // add metadata to distributed cache
+ attachKylinPropsAndMetadata(cube, job.getConfiguration());
+
+ String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
+ HTable htable = new HTable(conf, tableName);
+
+ //Automatic config !
+ HFileOutputFormat.configureIncrementalLoad(job, htable);
+
+ // set block replication to 3 for hfiles
+ conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
+
+ this.deletePath(job.getConfiguration(), output);
+
+ return waitForCompletion(job);
+ } catch (Exception e) {
+ printUsage(options);
+ log.error(e.getLocalizedMessage(), e);
+ return 2;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int exitCode = ToolRunner.run(new CubeHFileJob(), args);
+ System.exit(exitCode);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper.java
new file mode 100644
index 0000000..4f9b5c8
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.kv.RowConstants;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.HBaseColumnDesc;
+import com.kylinolap.metadata.model.cube.HBaseColumnFamilyDesc;
+import com.kylinolap.metadata.model.cube.MeasureDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class CubeHFileMapper extends Mapper<Text, Text, ImmutableBytesWritable, KeyValue> {
+
+ ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
+
+ String cubeName;
+ CubeDesc cubeDesc;
+
+ MeasureCodec inputCodec;
+ Object[] inputMeasures;
+ List<KeyValueCreator> keyValueCreators;
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
+
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
+
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+ cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();
+
+ inputCodec = new MeasureCodec(cubeDesc.getMeasures());
+ inputMeasures = new Object[cubeDesc.getMeasures().size()];
+ keyValueCreators = Lists.newArrayList();
+
+ for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
+ for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
+ keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
+ }
+ }
+ }
+
+ @Override
+ public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
+ outputKey.set(key.getBytes(), 0, key.getLength());
+ KeyValue outputValue;
+
+ int n = keyValueCreators.size();
+ if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for
+ // simple full copy
+
+ outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength());
+ context.write(outputKey, outputValue);
+
+ } else { // normal (complex) case that distributes measures to multiple
+ // HBase columns
+
+ inputCodec.decode(value, inputMeasures);
+
+ for (int i = 0; i < n; i++) {
+ outputValue = keyValueCreators.get(i).create(key, inputMeasures);
+ context.write(outputKey, outputValue);
+ }
+ }
+ }
+
+ class KeyValueCreator {
+ byte[] cfBytes;
+ byte[] qBytes;
+ long timestamp;
+
+ int[] refIndex;
+ MeasureDesc[] refMeasures;
+
+ MeasureCodec codec;
+ Object[] colValues;
+ ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+
+ boolean isFullCopy;
+
+ public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {
+
+ cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
+ qBytes = Bytes.toBytes(colDesc.getQualifier());
+ timestamp = System.currentTimeMillis();
+
+ List<MeasureDesc> measures = cubeDesc.getMeasures();
+ String[] measureNames = getMeasureNames(cubeDesc);
+ String[] refs = colDesc.getMeasureRefs();
+
+ refIndex = new int[refs.length];
+ refMeasures = new MeasureDesc[refs.length];
+ for (int i = 0; i < refs.length; i++) {
+ refIndex[i] = indexOf(measureNames, refs[i]);
+ refMeasures[i] = measures.get(refIndex[i]);
+ }
+
+ codec = new MeasureCodec(refMeasures);
+ colValues = new Object[refs.length];
+
+ isFullCopy = true;
+ for (int i = 0; i < measures.size(); i++) {
+ if (refIndex.length <= i || refIndex[i] != i)
+ isFullCopy = false;
+ }
+ }
+
+ public KeyValue create(Text key, Object[] measureValues) {
+ for (int i = 0; i < colValues.length; i++) {
+ colValues[i] = measureValues[refIndex[i]];
+ }
+
+ valueBuf.clear();
+ codec.encode(colValues, valueBuf);
+
+ return create(key, valueBuf.array(), 0, valueBuf.position());
+ }
+
+ public KeyValue create(Text key, byte[] value, int voffset, int vlen) {
+ return new KeyValue(key.getBytes(), 0, key.getLength(), //
+ cfBytes, 0, cfBytes.length, //
+ qBytes, 0, qBytes.length, //
+ timestamp, Type.Put, //
+ value, voffset, vlen);
+ }
+
+ private int indexOf(String[] measureNames, String ref) {
+ for (int i = 0; i < measureNames.length; i++)
+ if (measureNames[i].equalsIgnoreCase(ref))
+ return i;
+
+ throw new IllegalArgumentException("Measure '" + ref + "' not found in " + Arrays.toString(measureNames));
+ }
+
+ private String[] getMeasureNames(CubeDesc cubeDesc) {
+ List<MeasureDesc> measures = cubeDesc.getMeasures();
+ String[] result = new String[measures.size()];
+ for (int i = 0; i < measures.size(); i++)
+ result[i] = measures.get(i).getName();
+ return result;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidJob.java
new file mode 100644
index 0000000..687fe10
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidJob.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.cuboid.CuboidCLI;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.exception.JobException;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author ysong1
+ */
+public class CuboidJob extends AbstractHadoopJob {
+
+ protected static final Logger log = LoggerFactory.getLogger(CuboidJob.class);
+ private static final String MAPRED_REDUCE_TASKS = "mapred.reduce.tasks";
+
+ @SuppressWarnings("rawtypes")
+ private Class<? extends Mapper> mapperClass;
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_JOB_NAME);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_SEGMENT_NAME);
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ options.addOption(OPTION_NCUBOID_LEVEL);
+ options.addOption(OPTION_INPUT_FORMAT);
+ parseOptions(options, args);
+
+ Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+ String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+ int nCuboidLevel = Integer.parseInt(getOptionValue(OPTION_NCUBOID_LEVEL));
+ String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
+
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+
+ job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+ System.out.println("Starting: " + job.getJobName());
+ FileInputFormat.setInputPaths(job, input);
+
+ File jarFile = new File(config.getKylinJobJarPath());
+ if (jarFile.exists()) {
+ job.setJar(config.getKylinJobJarPath());
+ } else {
+ job.setJarByClass(this.getClass());
+ }
+
+ // Mapper
+ if (this.mapperClass == null) {
+ throw new Exception("Mapper class is not set!");
+ }
+
+ boolean isInputTextFormat = false;
+ if (hasOption(OPTION_INPUT_FORMAT) && ("textinputformat".equalsIgnoreCase(getOptionValue(OPTION_INPUT_FORMAT)))) {
+ isInputTextFormat = true;
+ }
+
+ if (isInputTextFormat) {
+ job.setInputFormatClass(TextInputFormat.class);
+
+ } else {
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ }
+ job.setMapperClass(this.mapperClass);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Text.class);
+ job.setCombinerClass(CuboidReducer.class); // for base cuboid shuffle skew, some rowkey aggregates far more records than others
+
+ // Reducer
+ job.setReducerClass(CuboidReducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ FileOutputFormat.setOutputPath(job, output);
+
+ // set job configuration
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+ // add metadata to distributed cache
+ attachKylinPropsAndMetadata(cube, job.getConfiguration());
+
+ setReduceTaskNum(job, config, cubeName, nCuboidLevel);
+
+ this.deletePath(job.getConfiguration(), output);
+
+ return waitForCompletion(job);
+ } catch (Exception e) {
+ printUsage(options);
+ log.error(e.getLocalizedMessage(), e);
+ return 2;
+ }
+ }
+
+ protected void setReduceTaskNum(Job job, KylinConfig config, String cubeName, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
+ Configuration jobConf = job.getConfiguration();
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
+ CubeDesc cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
+
+ double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
+ double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
+
+ // total map input MB
+ double totalMapInputMB = this.getTotalMapInputMB();
+
+ // output / input ratio
+ int preLevelCuboids, thisLevelCuboids;
+ if (level == 0) { // base cuboid
+ preLevelCuboids = thisLevelCuboids = 1;
+ } else { // n-cuboid
+ int[] allLevelCount = CuboidCLI.calculateAllLevelCount(cubeDesc);
+ preLevelCuboids = allLevelCount[level - 1];
+ thisLevelCuboids = allLevelCount[level];
+ }
+
+ // total reduce input MB
+ double totalReduceInputMB = totalMapInputMB * thisLevelCuboids / preLevelCuboids;
+
+ // number of reduce tasks
+ int numReduceTasks = (int) Math.round(totalReduceInputMB / perReduceInputMB * reduceCountRatio);
+
+ // adjust reducer number for cube which has DISTINCT_COUNT measures for
+ // better performance
+ if (cubeDesc.hasHolisticCountDistinctMeasures()) {
+ numReduceTasks = numReduceTasks * 4;
+ }
+
+ // at least 1 reducer
+ numReduceTasks = Math.max(1, numReduceTasks);
+ // no more than 5000 reducer by default
+ numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
+
+ jobConf.setInt(MAPRED_REDUCE_TASKS, numReduceTasks);
+
+ System.out.println("Having total map input MB " + Math.round(totalMapInputMB));
+ System.out.println("Having level " + level + ", pre-level cuboids " + preLevelCuboids + ", this level cuboids " + thisLevelCuboids);
+ System.out.println("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio);
+ System.out.println("Setting " + MAPRED_REDUCE_TASKS + "=" + numReduceTasks);
+ }
+
+ /**
+ * @param mapperClass
+ * the mapperClass to set
+ */
+ @SuppressWarnings("rawtypes")
+ public void setMapperClass(Class<? extends Mapper> mapperClass) {
+ this.mapperClass = mapperClass;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidReducer.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidReducer.java
new file mode 100644
index 0000000..50af652
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/CuboidReducer.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.kv.RowConstants;
+import com.kylinolap.cube.measure.MeasureAggregators;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.MeasureDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class CuboidReducer extends Reducer<Text, Text, Text, Text> {
+
+ private static final Logger logger = LoggerFactory.getLogger(CuboidReducer.class);
+
+ private String cubeName;
+ private CubeDesc cubeDesc;
+ private List<MeasureDesc> measuresDescs;
+
+ private MeasureCodec codec;
+ private MeasureAggregators aggs;
+
+ private int counter;
+ private Object[] input;
+ private Object[] result;
+
+ private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ private Text outputValue = new Text();
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
+
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
+
+ cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
+ measuresDescs = cubeDesc.getMeasures();
+
+ codec = new MeasureCodec(measuresDescs);
+ aggs = new MeasureAggregators(measuresDescs);
+
+ input = new Object[measuresDescs.size()];
+ result = new Object[measuresDescs.size()];
+ }
+
+ @Override
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+
+ aggs.reset();
+
+ for (Text value : values) {
+ codec.decode(value, input);
+ aggs.aggregate(input);
+ }
+ aggs.collectStates(result);
+
+ valueBuf.clear();
+ codec.encode(result, valueBuf);
+
+ outputValue.set(valueBuf.array(), 0, valueBuf.position());
+ context.write(key, outputValue);
+
+ counter++;
+ if (counter % BatchConstants.COUNTER_MAX == 0) {
+ logger.info("Handled " + counter + " records!");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsCombiner.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsCombiner.java
new file mode 100644
index 0000000..916dd9e
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsCombiner.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.util.HashSet;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import com.kylinolap.common.util.ByteArray;
+
+/**
+ * @author yangli9
+ */
+public class FactDistinctColumnsCombiner extends Reducer<ShortWritable, Text, ShortWritable, Text> {
+
+ private Text outputValue = new Text();
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ }
+
+ @Override
+ public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+
+ HashSet<ByteArray> set = new HashSet<ByteArray>();
+ for (Text textValue : values) {
+ ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
+ set.add(value);
+ }
+
+ for (ByteArray value : set) {
+ outputValue.set(value.data);
+ context.write(key, outputValue);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsJob.java
new file mode 100644
index 0000000..556f690
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+
+/**
+ * @author yangli9
+ */
+public class FactDistinctColumnsJob extends AbstractHadoopJob {
+ protected static final Logger log = LoggerFactory.getLogger(FactDistinctColumnsJob.class);
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_JOB_NAME);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_INPUT_FORMAT);
+ options.addOption(OPTION_OUTPUT_PATH);
+ parseOptions(options, args);
+
+ job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+ String cubeName = getOptionValue(OPTION_CUBE_NAME);
+ Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
+ String inputFormat = getOptionValue(OPTION_INPUT_FORMAT);
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+
+ // ----------------------------------------------------------------------------
+
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ System.out.println("Starting: " + job.getJobName());
+
+ setupMapInput(input, inputFormat);
+ setupReduceOutput(output);
+
+ // add metadata to distributed cache
+ CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ // CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
+ attachKylinPropsAndMetadata(cubeMgr.getCube(cubeName), job.getConfiguration());
+
+ return waitForCompletion(job);
+
+ } catch (Exception e) {
+ printUsage(options);
+ log.error(e.getLocalizedMessage(), e);
+ return 2;
+ }
+
+ }
+
+ private void setupMapInput(Path input, String inputFormat) throws IOException {
+ FileInputFormat.setInputPaths(job, input);
+
+ File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
+ if (JarFile.exists()) {
+ job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
+ } else {
+ job.setJarByClass(this.getClass());
+ }
+
+ if ("text".equalsIgnoreCase(inputFormat) || "textinputformat".equalsIgnoreCase(inputFormat)) {
+ job.setInputFormatClass(TextInputFormat.class);
+ } else {
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ }
+ job.setMapperClass(FactDistinctColumnsMapper.class);
+ job.setCombinerClass(FactDistinctColumnsCombiner.class);
+ job.setMapOutputKeyClass(ShortWritable.class);
+ job.setMapOutputValueClass(Text.class);
+ }
+
+ private void setupReduceOutput(Path output) throws IOException {
+ job.setReducerClass(FactDistinctColumnsReducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(Text.class);
+
+ FileOutputFormat.setOutputPath(job, output);
+ job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
+
+ job.setNumReduceTasks(1);
+
+ deletePath(job.getConfiguration(), output);
+ }
+
+ public static void main(String[] args) throws Exception {
+ FactDistinctColumnsJob job = new FactDistinctColumnsJob();
+ int exitCode = ToolRunner.run(job, args);
+ System.exit(exitCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java
new file mode 100644
index 0000000..236658d
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.common.BytesSplitter;
+import com.kylinolap.cube.common.SplittedBytes;
+import com.kylinolap.cube.cuboid.Cuboid;
+import com.kylinolap.dict.DictionaryManager;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.job.hadoop.hive.JoinedFlatTableDesc;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.RowKeyDesc;
+import com.kylinolap.metadata.model.cube.TblColRef;
+
+/**
+ * @author yangli9
+ */
+public class FactDistinctColumnsMapper<KEYIN> extends Mapper<KEYIN, Text, ShortWritable, Text> {
+
+ private String cubeName;
+ private CubeInstance cube;
+ private CubeDesc cubeDesc;
+ private int[] factDictCols;
+
+ private JoinedFlatTableDesc intermediateTableDesc;
+ private String intermediateTableRowDelimiter;
+ private byte byteRowDelimiter;
+ private BytesSplitter bytesSplitter;
+
+ private ShortWritable outputKey = new ShortWritable();
+ private Text outputValue = new Text();
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ intermediateTableRowDelimiter = conf.get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
+ byteRowDelimiter = intermediateTableRowDelimiter.getBytes("UTF-8")[0];
+ bytesSplitter = new BytesSplitter(200, 4096);
+
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
+ cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+ cube = CubeManager.getInstance(config).getCube(cubeName);
+ cubeDesc = cube.getDescriptor();
+ intermediateTableDesc = new JoinedFlatTableDesc(cubeDesc, null);
+
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ List<TblColRef> columns = baseCuboid.getColumns();
+
+ ArrayList<Integer> factDictCols = new ArrayList<Integer>();
+ RowKeyDesc rowkey = cubeDesc.getRowkey();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ for (int i = 0; i < columns.size(); i++) {
+ TblColRef col = columns.get(i);
+ if (rowkey.isUseDictionary(col) == false)
+ continue;
+
+ String scanTable = (String) dictMgr.decideSourceData(cubeDesc, col, null)[0];
+ if (cubeDesc.isFactTable(scanTable)) {
+ System.out.println(col + " -- " + i);
+ factDictCols.add(i);
+ }
+ }
+ this.factDictCols = new int[factDictCols.size()];
+ for (int i = 0; i < factDictCols.size(); i++)
+ this.factDictCols[i] = factDictCols.get(i);
+ }
+
+ @Override
+ public void map(KEYIN key, Text value, Context context) throws IOException, InterruptedException {
+
+ bytesSplitter.split(value.getBytes(), value.getLength(), byteRowDelimiter);
+ SplittedBytes[] splitBuffers = bytesSplitter.getSplitBuffers();
+
+ int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
+ for (int i : factDictCols) {
+ outputKey.set((short) i);
+ SplittedBytes bytes = splitBuffers[flatTableIndexes[i]];
+ outputValue.set(bytes.value, 0, bytes.length);
+ System.out.println(i + " -- " + outputValue);
+ context.write(outputKey, outputValue);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsReducer.java
new file mode 100644
index 0000000..e701781
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.ByteArray;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.cuboid.Cuboid;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.TblColRef;
+
+/**
+ * @author yangli9
+ */
+public class FactDistinctColumnsReducer extends Reducer<ShortWritable, Text, NullWritable, Text> {
+
+ private List<TblColRef> columnList = new ArrayList<TblColRef>();
+
+ @Override
+ protected void setup(Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
+ String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+ CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
+ CubeDesc cubeDesc = cube.getDescriptor();
+
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ columnList = baseCuboid.getColumns();
+ }
+
+ @Override
+ public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ TblColRef col = columnList.get(key.get());
+
+ HashSet<ByteArray> set = new HashSet<ByteArray>();
+ for (Text textValue : values) {
+ ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
+ set.add(value);
+ }
+
+ Configuration conf = context.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
+ FSDataOutputStream out = fs.create(new Path(outputPath, col.getName()));
+
+ try {
+ for (ByteArray value : set) {
+ out.write(value.data);
+ out.write('\n');
+ }
+ } finally {
+ out.close();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionJob.java
new file mode 100644
index 0000000..0c14abb
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionJob.java
@@ -0,0 +1,139 @@
+package com.kylinolap.job.hadoop.cube;
+
+///*
+// * Copyright 2013-2014 eBay Software Foundation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.kylinolap.index.cube;
+//
+//import org.apache.commons.cli.Options;
+//import org.apache.hadoop.fs.FileSystem;
+//import org.apache.hadoop.fs.Path;
+//import org.apache.hadoop.io.LongWritable;
+//import org.apache.hadoop.io.Text;
+//import org.apache.hadoop.mapreduce.Job;
+//import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+//import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+//import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+//import org.apache.hadoop.util.ToolRunner;
+//
+//import com.kylinolap.cube.CubeInstance;
+//import com.kylinolap.cube.CubeManager;
+//import com.kylinolap.cube.cuboid.Cuboid;
+//import com.kylinolap.cube.kv.AbstractRowKeyEncoder;
+//import com.kylinolap.cube.kv.RowKeyEncoder;
+//import com.kylinolap.index.AbstractHadoopJob;
+//import com.kylinolap.metadata.model.cube.CubeDesc;
+//
+///**
+// * @author xjiang
+// *
+// */
+//
+//public class KeyDistributionJob extends AbstractHadoopJob {
+//
+// public static final String JOB_TITLE = "Kylin Row Key Distribution Job";
+// public static final String KEY_HEADER_LENGTH = "key_header_length";
+// public static final String KEY_COLUMN_PERCENTAGE = "key_column_percentage";
+// public static final String KEY_SPLIT_NUMBER = "key_split_number";
+//
+// /* (non-Javadoc)
+// * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
+// */
+// @Override
+// public int run(String[] args) throws Exception {
+// Options options = new Options();
+//
+// try {
+// options.addOption(OPTION_INPUT_PATH);
+// options.addOption(OPTION_OUTPUT_PATH);
+// options.addOption(OPTION_METADATA_URL);
+// options.addOption(OPTION_CUBE_NAME);
+// options.addOption(OPTION_KEY_COLUMN_PERCENTAGE);
+// options.addOption(OPTION_KEY_SPLIT_NUMBER);
+// parseOptions(options, args);
+//
+// // start job
+// String jobName = JOB_TITLE + getOptionsAsString();
+// System.out.println("Starting: " + jobName);
+// Job job = Job.getInstanceFromEnv(getConf(), jobName);
+//
+// // set job configuration - basic
+// job.setJarByClass(getClass());
+// addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
+//
+// Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+// FileOutputFormat.setOutputPath(job, output);
+// //job.getConfiguration().set("dfs.block.size", "67108864");
+//
+// // set job configuration - key prefix size & key split number
+// String keyColumnPercentage = getOptionValue(OPTION_KEY_COLUMN_PERCENTAGE);
+// job.getConfiguration().set(KEY_COLUMN_PERCENTAGE, keyColumnPercentage);
+// String metadataUrl = validateMetadataUrl(getOptionValue(OPTION_METADATA_URL));
+// String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+// int keyHeaderLen = getKeyHeaderLength(metadataUrl, cubeName);
+// job.getConfiguration().set(KEY_HEADER_LENGTH, String.valueOf(keyHeaderLen));
+// job.getConfiguration().set(KEY_SPLIT_NUMBER, getOptionValue(OPTION_KEY_SPLIT_NUMBER));
+//
+// // Mapper
+// job.setInputFormatClass(SequenceFileInputFormat.class);
+// job.setMapperClass(KeyDistributionMapper.class);
+// job.setMapOutputKeyClass(Text.class);
+// job.setMapOutputValueClass(LongWritable.class);
+//
+// // Combiner, not needed any more as mapper now does the groping
+// //job.setCombinerClass(KeyDistributionCombiner.class);
+//
+// // Reducer - only one
+// job.setReducerClass(KeyDistributionReducer.class);
+// // use sequence file as output
+// job.setOutputFormatClass(SequenceFileOutputFormat.class);
+// // key is text
+// job.setOutputKeyClass(Text.class);
+// // value is long
+// job.setOutputValueClass(LongWritable.class);
+// job.setNumReduceTasks(1);
+//
+// FileSystem fs = FileSystem.get(job.getConfiguration());
+// if (fs.exists(output))
+// fs.delete(output, true);
+//
+// return waitForCompletion(job);
+// } catch (Exception e) {
+// printUsage(options);
+// e.printStackTrace(System.err);
+// return 2;
+// }
+// }
+//
+// private int getKeyHeaderLength(String metadataUrl, String cubeName) {
+// CubeManager cubeMgr = CubeManager.getInstanceFromEnv(metadataUrl);
+// CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
+// CubeDesc cubeDesc = cubeInstance.getDescriptor();
+// long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+// Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+// RowKeyEncoder rowKeyEncoder =
+// (RowKeyEncoder) AbstractRowKeyEncoder.createInstance(cubeInstance.getTheOnlySegment(),
+// baseCuboid);
+//
+// return rowKeyEncoder.getHeaderLength();
+//
+// }
+//
+// public static void main(String[] args) throws Exception {
+// int exitCode = ToolRunner.run(new KeyDistributionJob(), args);
+// System.exit(exitCode);
+// }
+// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapper.java
new file mode 100644
index 0000000..037c3d9
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapper.java
@@ -0,0 +1,104 @@
+package com.kylinolap.job.hadoop.cube;
+
+///*
+// * Copyright 2013-2014 eBay Software Foundation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.kylinolap.index.cube;
+//
+//import java.io.IOException;
+//
+//import org.apache.hadoop.io.LongWritable;
+//import org.apache.hadoop.io.Text;
+//import org.apache.hadoop.mapreduce.Mapper;
+//
+///**
+// * @author xjiang
+// *
+// */
+//public class KeyDistributionMapper extends Mapper<Text, Text, Text, LongWritable> {
+//
+// private int headerLength;
+//
+// private Text currentKey;
+// private long outputLong;
+// private Text outputKey;
+// private LongWritable outputValue;
+// private int columnPercentage;
+// private int allRowCount;
+//
+// @Override
+// protected void setup(Context context) throws IOException {
+// String percentStr = context.getConfiguration().get(KeyDistributionJob.KEY_COLUMN_PERCENTAGE);
+// this.columnPercentage = Integer.valueOf(percentStr).intValue();
+// if (this.columnPercentage <= 0 || this.columnPercentage >= 100) {
+// this.columnPercentage = 20;
+// }
+// String headerLenStr = context.getConfiguration().get(KeyDistributionJob.KEY_HEADER_LENGTH);
+// this.headerLength = Integer.valueOf(headerLenStr).intValue();
+//
+// currentKey = new Text();
+// outputLong = 0;
+// outputKey = new Text();
+// outputValue = new LongWritable(1);
+// allRowCount = 0;
+// }
+//
+// @Override
+// protected void cleanup(Context context) throws IOException, InterruptedException {
+// emit(context); // emit the last holding record
+//
+// byte[] zerokey = new byte[] { 0 };
+// outputKey.set(zerokey);
+// outputValue.set(allRowCount);
+// context.write(outputKey, outputValue);
+// }
+//
+// @Override
+// public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
+// byte[] bytes = key.getBytes();
+// int columnLength = bytes.length - this.headerLength;
+// int columnPrefixLen = columnLength * this.columnPercentage / 100;
+// if (columnPrefixLen == 0 && columnLength > 0) {
+// columnPrefixLen = 1;
+// }
+// if (columnPrefixLen > 0) {
+// currentKey.set(bytes, 0, this.headerLength + columnPrefixLen);
+// } else {
+// currentKey.set(bytes);
+// }
+//
+// allRowCount++;
+//
+// if (outputKey.getLength() == 0) { // first record
+// outputKey.set(currentKey);
+// outputLong = 1;
+// } else if (outputKey.equals(currentKey)) { // same key, note input is sorted
+// outputLong++;
+// } else { // the next key
+// emit(context);
+// outputKey.set(currentKey);
+// outputLong = 1;
+// }
+// }
+//
+// private void emit(Context context) throws IOException, InterruptedException {
+// if (outputLong == 0)
+// return;
+//
+// outputValue.set(outputLong);
+// context.write(outputKey, outputValue);
+// }
+// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionReducer.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionReducer.java
new file mode 100644
index 0000000..e0edc20
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/KeyDistributionReducer.java
@@ -0,0 +1,93 @@
+package com.kylinolap.job.hadoop.cube;
+
+///*
+// * Copyright 2013-2014 eBay Software Foundation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.kylinolap.index.cube;
+//
+//import java.io.IOException;
+//
+//import org.apache.hadoop.io.LongWritable;
+//import org.apache.hadoop.io.Text;
+//import org.apache.hadoop.mapreduce.Reducer;
+//import org.apache.hadoop.util.StringUtils;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//
+///**
+// * @author xjiang
+// *
+// */
+//public class KeyDistributionReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
+//
+// private static final Logger logger = LoggerFactory.getLogger(KeyDistributionReducer.class);
+//
+// private LongWritable outputValue;
+// private boolean isTotalCount;
+// private long totalCount;
+// private int splitNumber;
+// private long splitQuota;
+// private long splitRemain;
+//
+// @Override
+// protected void setup(Context context) throws IOException, InterruptedException {
+// String splitStr = context.getConfiguration().get(KeyDistributionJob.KEY_SPLIT_NUMBER);
+// splitNumber = Integer.valueOf(splitStr).intValue();
+// outputValue = new LongWritable();
+// isTotalCount = true;
+// totalCount = 0;
+// splitQuota = 0;
+// splitRemain = 0;
+// }
+//
+// @Override
+// protected void cleanup(Context context) throws IOException, InterruptedException {
+// logger.info("---------------");
+// long splitCount = splitQuota - splitRemain;
+// logger.info("Total Count = " + totalCount + ", Left Count = " + splitCount);
+// }
+//
+// @Override
+// public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException,
+// InterruptedException {
+//
+// // calculate split quota
+// if (isTotalCount) {
+// for (LongWritable count : values) {
+// totalCount += count.get();
+// }
+// splitQuota = totalCount / splitNumber;
+// splitRemain = splitQuota;
+// isTotalCount = false;
+// return;
+// }
+//
+// // output key when split quota is used up
+// for (LongWritable count : values) {
+// splitRemain -= count.get();
+// }
+// if (splitRemain <= 0) {
+// long splitCount = splitQuota - splitRemain;
+// String hexKey = StringUtils.byteToHexString(key.getBytes());
+// logger.info(hexKey + "\t\t" + splitCount);
+//
+// outputValue.set(splitCount);
+// context.write(key, outputValue);
+// splitRemain = splitQuota;
+// }
+//
+// }
+// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidJob.java
new file mode 100644
index 0000000..d97e135
--- /dev/null
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidJob.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.File;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.job.constant.BatchConstants;
+
+/**
+ * @author ysong1
+ */
+public class MergeCuboidJob extends CuboidJob {
+
+ private static final Logger log = LoggerFactory.getLogger(MergeCuboidJob.class);
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ try {
+ options.addOption(OPTION_JOB_NAME);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_SEGMENT_NAME);
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ parseOptions(options, args);
+
+ String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
+ String segmentName = getOptionValue(OPTION_SEGMENT_NAME).toUpperCase();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+ // CubeSegment cubeSeg = cubeMgr.findSegment(cube, segmentName);
+
+ // start job
+ String jobName = getOptionValue(OPTION_JOB_NAME);
+ System.out.println("Starting: " + jobName);
+ job = Job.getInstance(getConf(), jobName);
+
+ // set job configuration - basic
+ File JarFile = new File(config.getKylinJobJarPath());
+ if (JarFile.exists()) {
+ job.setJar(config.getKylinJobJarPath());
+ } else {
+ job.setJarByClass(this.getClass());
+ }
+
+ // setJobJar(job);
+ addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
+
+ Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+ FileOutputFormat.setOutputPath(job, output);
+
+ // Mapper
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setMapperClass(MergeCuboidMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Text.class);
+
+ // Reducer - only one
+ job.setReducerClass(CuboidReducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // set job configuration
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+
+ // add metadata to distributed cache
+ attachKylinPropsAndMetadata(cube, job.getConfiguration());
+
+ setReduceTaskNum(job, config, cubeName, 0);
+
+ this.deletePath(job.getConfiguration(), output);
+
+ return waitForCompletion(job);
+ } catch (Exception e) {
+ printUsage(options);
+ log.error(e.getLocalizedMessage(), e);
+ return 2;
+ }
+ }
+
+}