You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:49:40 UTC
[01/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Repository: kylin
Updated Branches:
refs/heads/document [created] 6b6aa3135
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/jarfile/SampleBadJavaProgram.jarfile
----------------------------------------------------------------------
diff --git a/job/src/test/resources/jarfile/SampleBadJavaProgram.jarfile b/job/src/test/resources/jarfile/SampleBadJavaProgram.jarfile
deleted file mode 100644
index 75a43d5..0000000
Binary files a/job/src/test/resources/jarfile/SampleBadJavaProgram.jarfile and /dev/null differ
[47/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
deleted file mode 100644
index 59cb86b..0000000
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ /dev/null
@@ -1,775 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.SortedSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.restclient.RestClient;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.Log4jConfigurer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- */
-public class KylinConfig {
-
- /*
- * NO NEED TO DEFINE PUBLIC CONSTANTS FOR KEY NAMES!
- *
- * Kylin configuration is exposed as public methods. A client never need to access key names directly.
- * Feel free to hard code key names, as long as they are encapsulated in this class. It reads better!
- */
-
- public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
-
- public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
-
- public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
-
- public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone";
-
- public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
-
- public static final String KYLIN_JOB_YARN_APP_REST_CHECK_URL = "kylin.job.yarn.app.rest.check.status.url";
-
- public static final String KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS = "kylin.job.yarn.app.rest.check.interval.seconds";
-
- public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir";
-
- public static final String HIVE_DATABASE_FOR_INTERMEDIATE_TABLE = "kylin.job.hive.database.for.intermediatetable";
-
- public static final String HIVE_TABLE_LOCATION_PREFIX = "hive.table.location.";
-
- public static final String KYLIN_JOB_REMOTE_CLI_PASSWORD = "kylin.job.remote.cli.password";
-
- public static final String KYLIN_JOB_REMOTE_CLI_USERNAME = "kylin.job.remote.cli.username";
-
- public static final String KYLIN_JOB_REMOTE_CLI_HOSTNAME = "kylin.job.remote.cli.hostname";
-
- public static final String KYLIN_JOB_REMOTE_CLI_WORKING_DIR = "kylin.job.remote.cli.working.dir";
-
- public static final String KYLIN_JOB_CMD_EXTRA_ARGS = "kylin.job.cmd.extra.args";
-
- public static final String KYLIN_GET_JOB_STATUS_WITH_KERBEROS = "kylin.job.status.with.kerberos";
- /**
- * Toggle to indicate whether to use hive for table flattening. Default
- * true.
- */
- public static final String KYLIN_JOB_RUN_AS_REMOTE_CMD = "kylin.job.run.as.remote.cmd";
-
- public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO = "kylin.job.mapreduce.default.reduce.count.ratio";
-
- public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB = "kylin.job.mapreduce.default.reduce.input.mb";
-
- public static final String KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER = "kylin.job.mapreduce.max.reducer.number";
-
- public static final String KYLIN_JOB_JAR = "kylin.job.jar";
-
- public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
-
- public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
-
- public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir";
-
- public static final String KYLIN_HBASE_CLUSTER_FS = "kylin.hbase.cluster.fs";
-
- /**
- * Kylin properties file
- */
- public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
-
- public static final String MAIL_ENABLED = "mail.enabled";
-
- public static final String MAIL_HOST = "mail.host";
-
- public static final String MAIL_USERNAME = "mail.username";
-
- public static final String MAIL_PASSWORD = "mail.password";
-
- public static final String MAIL_SENDER = "mail.sender";
-
- public static final String KYLIN_HOME = "KYLIN_HOME";
-
- public static final String KYLIN_CONF = "KYLIN_CONF";
-
- private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class);
-
- public static final String VERSION = "${project.version}";
-
- public static final String HTABLE_DEFAULT_COMPRESSION_CODEC = "kylin.hbase.default.compression.codec";
-
- public static final String HBASE_REGION_CUT_SMALL = "kylin.hbase.region.cut.small";
- public static final String HBASE_REGION_CUT_MEDIUM = "kylin.hbase.region.cut.medium";
- public static final String HBASE_REGION_CUT_LARGE = "kylin.hbase.region.cut.large";
-
- public static final String HBASE_REGION_COUNT_MIN = "kylin.hbase.region.count.min";
- public static final String HBASE_REGION_COUNT_MAX = "kylin.hbase.region.count.max";
-
- // static cached instances
- private static KylinConfig ENV_INSTANCE = null;
-
- public static KylinConfig getInstanceFromEnv() {
- if (ENV_INSTANCE == null) {
- try {
- KylinConfig config = loadKylinConfig();
- ENV_INSTANCE = config;
- } catch (IllegalArgumentException e) {
- throw new IllegalStateException("Failed to find KylinConfig ", e);
- }
- }
- return ENV_INSTANCE;
- }
-
- public static void destoryInstance() {
- ENV_INSTANCE = null;
- }
-
- public static enum UriType {
- PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
- }
-
- private static UriType decideUriType(String metaUri) {
-
- try {
- File file = new File(metaUri);
- if (file.exists() || metaUri.contains("/")) {
- if (file.exists() == false) {
- file.mkdirs();
- }
- if (file.isDirectory()) {
- return UriType.LOCAL_FOLDER;
- } else if (file.isFile()) {
- if (file.getName().equalsIgnoreCase(KYLIN_CONF_PROPERTIES_FILE)) {
- return UriType.PROPERTIES_FILE;
- } else {
- throw new IllegalStateException("Metadata uri : " + metaUri + " is a local file but not kylin.properties");
- }
- } else {
- throw new IllegalStateException("Metadata uri : " + metaUri + " looks like a file but it's neither a file nor a directory");
- }
- } else {
- if (RestClient.matchFullRestPattern(metaUri))
- return UriType.REST_ADDR;
- else
- throw new IllegalStateException("Metadata uri : " + metaUri + " is not a valid REST URI address");
- }
- } catch (Exception e) {
- logger.info(e.getLocalizedMessage());
- throw new IllegalStateException("Metadata uri : " + metaUri + " is not recognized");
- }
- }
-
- public static KylinConfig createInstanceFromUri(String uri) {
- /**
- * --hbase: 1. PROPERTIES_FILE: path to kylin.properties 2. REST_ADDR:
- * rest service resource, format: user:password@host:port --local: 1.
- * LOCAL_FOLDER: path to resource folder
- */
- UriType uriType = decideUriType(uri);
- logger.info("The URI " + uri + " is recognized as " + uriType);
-
- if (uriType == UriType.LOCAL_FOLDER) {
- KylinConfig config = new KylinConfig();
- config.setMetadataUrl(uri);
- return config;
- } else if (uriType == UriType.PROPERTIES_FILE) {
- KylinConfig config;
- try {
- config = new KylinConfig();
- InputStream is = new FileInputStream(uri);
- config.reloadKylinConfig(is);
- is.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return config;
- } else {// rest_addr
- try {
- KylinConfig config = new KylinConfig();
- RestClient client = new RestClient(uri);
- String propertyText = client.getKylinProperties();
- InputStream is = IOUtils.toInputStream(propertyText);
- config.reloadKylinConfig(is);
- is.close();
- return config;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public static KylinConfig getKylinConfigFromInputStream(InputStream is) {
- KylinConfig config = new KylinConfig();
- config.reloadKylinConfig(is);
- return config;
- }
-
- // ============================================================================
-
- /**
- * Find config from environment. The Search process: 1. Check the
- * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties
- *
- * @return
- */
- private static KylinConfig loadKylinConfig() {
- Log4jConfigurer.initLogger();
-
- InputStream is = getKylinPropertiesAsInputSteam();
- if (is == null) {
- throw new IllegalArgumentException("Failed to load kylin config");
- }
- KylinConfig config = new KylinConfig();
- config.reloadKylinConfig(is);
-
- return config;
- }
-
- private PropertiesConfiguration kylinConfig = new PropertiesConfiguration();
-
- private String metadataUrl;
- private String storageUrl;
-
- public CliCommandExecutor getCliCommandExecutor() throws IOException {
- CliCommandExecutor exec = new CliCommandExecutor();
- if (getRunAsRemoteCommand()) {
- exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
- }
- return exec;
- }
-
- // ============================================================================
-
- public boolean isHiveReroutingEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.route.hive.enabled", "false"));
- }
-
- public String getHiveRerouteUrl() {
- return getOptional("kylin.route.hive.url", "jdbc:hive2://");
- }
-
- public String getHiveRerouteUsername() {
- return getOptional("kylin.route.hive.username", "");
- }
-
- public String getHiveReroutePassword() {
- return getOptional("kylin.route.hive.password", "");
- }
-
- public String getStorageUrl() {
- return storageUrl;
- }
-
- /** Use the hive reroute feature instead */
- @Deprecated
- public String getHiveUrl() {
- return getOptional("hive.url", "");
- }
-
- /** Use the hive reroute feature instead */
- @Deprecated
- public String getHiveUser() {
- return getOptional("hive.user", "");
- }
-
- /** Use the hive reroute feature instead */
- @Deprecated
- public String getHivePassword() {
- return getOptional("hive.password", "");
- }
-
- public String getHdfsWorkingDirectory() {
- String root = getRequired(KYLIN_HDFS_WORKING_DIR);
- if (!root.endsWith("/")) {
- root += "/";
- }
- return new StringBuffer(root)
- .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-'))
- .append("/")
- .toString();
- }
-
- public String getHBaseClusterFs() {
- return getOptional(KYLIN_HBASE_CLUSTER_FS, "");
- }
-
- public String getKylinJobLogDir() {
- return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs");
- }
-
- public String getKylinJobJarPath() {
- final String jobJar = getOptional(KYLIN_JOB_JAR);
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
- }
- return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
- }
-
- public String getKylinJobMRLibDir() {
- return getOptional("kylin.job.mr.lib.dir", "");
- }
-
- public void overrideKylinJobJarPath(String path) {
- logger.info("override " + KYLIN_JOB_JAR + " to " + path);
- System.setProperty(KYLIN_JOB_JAR, path);
- }
-
- private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
- private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
-
- public String getCoprocessorLocalJar() {
- final String coprocessorJar = getOptional(COPROCESSOR_LOCAL_JAR);
- if (StringUtils.isNotEmpty(coprocessorJar)) {
- return coprocessorJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
- }
- return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
- }
-
- private static String getFileName(String homePath, Pattern pattern) {
- File home = new File(homePath);
- SortedSet<String> files = Sets.newTreeSet();
- if (home.exists() && home.isDirectory()) {
- for (File file : home.listFiles()) {
- final Matcher matcher = pattern.matcher(file.getName());
- if (matcher.matches()) {
- files.add(file.getAbsolutePath());
- }
- }
- }
- if (files.isEmpty()) {
- throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
- } else {
- return files.last();
- }
- }
-
- public void overrideCoprocessorLocalJar(String path) {
- logger.info("override " + COPROCESSOR_LOCAL_JAR + " to " + path);
- System.setProperty(COPROCESSOR_LOCAL_JAR, path);
- }
-
- public double getDefaultHadoopJobReducerInputMB() {
- return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500"));
- }
-
- public double getDefaultHadoopJobReducerCountRatio() {
- return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO, "1.0"));
- }
-
- public int getHadoopJobMaxReducerNumber() {
- return Integer.parseInt(getOptional(KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER, "5000"));
- }
-
- public boolean getRunAsRemoteCommand() {
- return Boolean.parseBoolean(getOptional(KYLIN_JOB_RUN_AS_REMOTE_CMD));
- }
-
- public String getRemoteHadoopCliHostname() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_HOSTNAME);
- }
-
- public String getRemoteHadoopCliUsername() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_USERNAME);
- }
-
- public String getRemoteHadoopCliPassword() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_PASSWORD);
- }
-
- public String getCliWorkingDir() {
- return getOptional(KYLIN_JOB_REMOTE_CLI_WORKING_DIR);
- }
-
- public String getMapReduceCmdExtraArgs() {
- return getOptional(KYLIN_JOB_CMD_EXTRA_ARGS);
- }
-
- public boolean getKylinUseKerberosAuth() {
- return Boolean.valueOf(getOptional(KYLIN_GET_JOB_STATUS_WITH_KERBEROS, "false"));
- }
-
- public String getOverrideHiveTableLocation(String table) {
- return getOptional(HIVE_TABLE_LOCATION_PREFIX + table.toUpperCase());
- }
-
- public String getTempHDFSDir() {
- return getOptional(KYLIN_TMP_HDFS_DIR, "/tmp/kylin");
- }
-
- public String getYarnStatusCheckUrl() {
- return getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_URL, null);
- }
-
- public int getYarnStatusCheckIntervalSeconds() {
- return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS, "60"));
- }
-
- public int getMaxConcurrentJobLimit() {
- return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10"));
- }
-
- public String getTimeZone() {
- return getOptional(KYLIN_REST_TIMEZONE, "PST");
- }
-
- public String[] getRestServers() {
- return getOptionalStringArray(KYLIN_REST_SERVERS);
- }
-
- public String getAdminDls() {
- return getOptional("kylin.job.admin.dls", null);
- }
-
- public long getJobStepTimeout() {
- return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60)));
- }
-
- public String getServerMode() {
- return this.getOptional("kylin.server.mode", "all");
- }
-
- public int getCachedDictMaxEntrySize() {
- return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000"));
- }
-
-
- public int getDictionaryMaxCardinality() {
- return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
- }
-
- public int getTableSnapshotMaxMB() {
- return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", "300"));
- }
-
- public int getScanThreshold() {
- return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
- }
-
- public boolean getQueryRunLocalCoprocessor() {
- return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
- }
-
- public Long getQueryDurationCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
- }
-
- public Long getQueryScanCountCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024)));
- }
-
- public boolean isQuerySecureEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true"));
- }
-
- public boolean isQueryCacheEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
- }
-
- public long getQueryMemBudget() {
- return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024)));
- }
-
- public int getHBaseKeyValueSize() {
- return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
- }
-
- public int getHBaseScanCacheRows() {
- return Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
- }
-
- public int getHBaseScanMaxResultSize() {
- return Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 1024 * 1024))); // 5 MB
- }
-
- public boolean isQueryIgnoreUnknownFunction() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false"));
- }
-
- public String getHbaseDefaultCompressionCodec() {
- return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, "");
-
- }
-
- private String getOptional(String prop) {
- final String property = System.getProperty(prop);
- return property != null ? property : kylinConfig.getString(prop);
- }
-
- private String[] getOptionalStringArray(String prop) {
- final String property = System.getProperty(prop);
- if (!StringUtils.isBlank(property))
- return property.split("\\s*,\\s*");
-
- return kylinConfig.getStringArray(prop);
- }
-
- private String getOptional(String prop, String dft) {
- final String property = System.getProperty(prop);
- return property != null ? property : kylinConfig.getString(prop, dft);
- }
-
- private String getRequired(String prop) {
- final String property = System.getProperty(prop);
- if (property != null) {
- return property;
- }
- String r = kylinConfig.getString(prop);
- if (StringUtils.isEmpty(r)) {
- throw new IllegalArgumentException("missing '" + prop + "' in conf/kylin_instance.properties");
- }
- return r;
- }
-
- void reloadKylinConfig(InputStream is) {
- PropertiesConfiguration config = new PropertiesConfiguration();
- try {
- config.load(is);
- } catch (ConfigurationException e) {
- throw new RuntimeException("Cannot load kylin config.", e);
- } finally {
- try {
- is.close();
- } catch (IOException e) {
- logger.error("Failed to close inputstream.", e);
- }
- }
- this.kylinConfig = config;
- this.metadataUrl = getOptional(KYLIN_METADATA_URL);
- this.storageUrl = getOptional(KYLIN_STORAGE_URL);
- }
-
- public void writeProperties(File file) throws IOException {
- try {
- kylinConfig.save(file);
- } catch (ConfigurationException ex) {
- throw new IOException("Error writing KylinConfig to " + file, ex);
- }
- }
-
- public static String getKylinHome() {
- String kylinHome = System.getenv(KYLIN_HOME);
- if (StringUtils.isEmpty(kylinHome)) {
- logger.warn("KYLIN_HOME was not set");
- return kylinHome;
- }
- return kylinHome;
- }
-
- public void printProperties() throws IOException {
- try {
- kylinConfig.save(System.out);
- } catch (ConfigurationException ex) {
- throw new IOException("Error printing KylinConfig", ex);
- }
- }
-
- private static File getKylinProperties() {
- String kylinConfHome = System.getProperty(KYLIN_CONF);
- if (!StringUtils.isEmpty(kylinConfHome)) {
- logger.info("Use KYLIN_CONF=" + kylinConfHome);
- return getKylinPropertiesFile(kylinConfHome);
- }
-
- logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable");
-
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome))
- throw new RuntimeException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
-
- String path = kylinHome + File.separator + "conf";
- return getKylinPropertiesFile(path);
-
- }
-
- public static InputStream getKylinPropertiesAsInputSteam() {
- File propFile = getKylinProperties();
- if (propFile == null || !propFile.exists()) {
- logger.error("fail to locate kylin.properties");
- throw new RuntimeException("fail to locate kylin.properties");
- }
-
- File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
- if (overrideFile.exists()) {
- try {
- PropertiesConfiguration conf = new PropertiesConfiguration();
- conf.load(propFile);
- PropertiesConfiguration override = new PropertiesConfiguration();
- override.load(overrideFile);
- conf.copy(override);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- conf.save(bout);
- return new ByteArrayInputStream(bout.toByteArray());
- } catch (ConfigurationException e) {
- throw new RuntimeException(e);
- }
- }
-
- try {
- return new FileInputStream(propFile);
- } catch (FileNotFoundException e) {
- logger.error("this should not happen");
- throw new RuntimeException(e);
- }
-
- }
-
- /**
- * Check if there is kylin.properties exist
- *
- * @param path
- * @return the properties file
- */
- private static File getKylinPropertiesFile(String path) {
- if (path == null) {
- return null;
- }
-
- return new File(path, KYLIN_CONF_PROPERTIES_FILE);
- }
-
- public String getMetadataUrl() {
- return metadataUrl;
- }
-
- public String getMetadataUrlPrefix() {
- String hbaseMetadataUrl = getMetadataUrl();
- String defaultPrefix = "kylin_metadata";
-
- if (hbaseMetadataUrl.indexOf("@hbase") > 0) {
- int cut = hbaseMetadataUrl.indexOf('@');
- String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
- return tmp;
- } else {
- return defaultPrefix;
- }
- }
-
- public void setMetadataUrl(String metadataUrl) {
- kylinConfig.setProperty(KYLIN_METADATA_URL, metadataUrl);
- this.metadataUrl = metadataUrl;
- }
-
- public void setStorageUrl(String storageUrl) {
- kylinConfig.setProperty(KYLIN_STORAGE_URL, storageUrl);
- this.storageUrl = storageUrl;
- }
-
- public String getHiveDatabaseForIntermediateTable() {
- return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default");
- }
-
- public void setRunAsRemoteCommand(String v) {
- kylinConfig.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
- }
-
- public void setRemoteHadoopCliHostname(String v) {
- kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_HOSTNAME, v);
- }
-
- public void setRemoteHadoopCliUsername(String v) {
- kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_USERNAME, v);
- }
-
- public void setRemoteHadoopCliPassword(String v) {
- kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v);
- }
-
- public int getHBaseRegionCountMin() {
- return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MIN, "1"));
- }
-
- public int getHBaseRegionCountMax() {
- return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MAX, "500"));
- }
-
- public int getHBaseRegionCut(String capacity) {
- String cut;
- switch (capacity) {
- case "SMALL":
- cut = getProperty(HBASE_REGION_CUT_SMALL, "10");
- break;
- case "MEDIUM":
- cut = getProperty(HBASE_REGION_CUT_MEDIUM, "20");
- break;
- case "LARGE":
- cut = getProperty(HBASE_REGION_CUT_LARGE, "100");
- break;
- default:
- throw new IllegalArgumentException("Capacity not recognized: " + capacity);
- }
-
- return Integer.valueOf(cut);
- }
-
- public String getProperty(String key, String defaultValue) {
- return kylinConfig.getString(key, defaultValue);
- }
-
- /**
- * Set a new key:value into the kylin config.
- *
- * @param key
- * @param value
- */
- public void setProperty(String key, String value) {
- logger.info("Kylin Config was updated with " + key + " : " + value);
- kylinConfig.setProperty(key, value);
- }
-
- public String getConfigAsString() throws IOException {
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- kylinConfig.save(baos);
- String content = baos.toString();
- return content;
- } catch (ConfigurationException ex) {
- throw new IOException("Error writing KylinConfig to String", ex);
- }
- }
-
- public String toString() {
- return getMetadataUrl();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
deleted file mode 100644
index 6095295..0000000
--- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.debug;
-
-import java.util.Map;
-
-/**
- */
-public class BackdoorToggles {
-
- private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
-
- public static void setToggles(Map<String, String> toggles) {
- _backdoorToggles.set(toggles);
- }
-
- public static String getObserverBehavior() {
- return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
- }
-
- public static boolean getDisableFuzzyKey() {
- return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
- }
-
- public static boolean getRunLocalCoprocessor() {
- return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
- }
-
- private static String getString(String key) {
- Map<String, String> toggles = _backdoorToggles.get();
- if (toggles == null) {
- return null;
- } else {
- return toggles.get(key);
- }
- }
-
- private static boolean getBoolean(String key) {
- return "true".equals(getString(key));
- }
-
- public static void cleanToggles() {
- _backdoorToggles.remove();
- }
-
- /**
- * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
- *
- *
- *
- example:
-
- "backdoorToggles": {
- "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true"
- }
-
- */
- public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
-
- /**
- * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
- *
- example:
-
- "backdoorToggles": {
- "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
- }
-
- */
- public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
-
- /**
- * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
- *
- example:
-
- "backdoorToggles": {
- "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true"
- }
-
- */
- public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR";
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
deleted file mode 100644
index d37d6db..0000000
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.hll;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-/**
- * About compression, test on HLLC data shows
- *
- * - LZF compression ratio is around 65%-80%, fast
- * - GZIP compression ratio is around 41%-46%, very slow
- *
- * @author yangli9
- */
-@SuppressWarnings("serial")
-public class HyperLogLogPlusCounter implements Serializable, Comparable<HyperLogLogPlusCounter> {
-
- private final int p;
- private final int m;
- private final HashFunction hashFunc;
- byte[] registers;
-
- public HyperLogLogPlusCounter() {
- this(10);
- }
-
- public HyperLogLogPlusCounter(int p) {
- this(p, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
- this(another.p, another.hashFunc);
- merge(another);
- }
-
- /** The larger p is, the more storage (2^p bytes), the better accuracy */
- private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
- this.p = p;
- this.m = 1 << p;//(int) Math.pow(2, p);
- this.hashFunc = hashFunc;
- this.registers = new byte[m];
- }
-
- public void clear() {
- byte zero = (byte) 0;
- Arrays.fill(registers, zero);
- }
-
- public void add(int value) {
- add(hashFunc.hashInt(value).asLong());
- }
-
- public void add(String value) {
- add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
- }
-
- public void add(byte[] value) {
- add(hashFunc.hashBytes(value).asLong());
- }
-
- public void add(byte[] value, int offset, int length) {
- add(hashFunc.hashBytes(value, offset, length).asLong());
- }
-
- protected void add(long hash) {
- int bucketMask = m - 1;
- int bucket = (int) (hash & bucketMask);
- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
-
- if (firstOnePos > registers[bucket])
- registers[bucket] = (byte) firstOnePos;
- }
-
- public void merge(HyperLogLogPlusCounter another) {
- assert this.p == another.p;
- assert this.hashFunc == another.hashFunc;
-
- for (int i = 0; i < m; i++) {
- if (registers[i] < another.registers[i])
- registers[i] = another.registers[i];
- }
- }
-
- public long getCountEstimate() {
- return new HLLCSnapshot(this).getCountEstimate();
- }
-
- public double getErrorRate() {
- return 1.04 / Math.sqrt(m);
- }
-
- private int size() {
- int size = 0;
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0)
- size++;
- }
- return size;
- }
-
- @Override
- public String toString() {
- return "" + getCountEstimate();
- }
-
- // ============================================================================
-
- // a memory efficient snapshot of HLL registers which can yield count
- // estimate later
- public static class HLLCSnapshot {
- byte p;
- double registerSum;
- int zeroBuckets;
-
- public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
- p = (byte) hllc.p;
- registerSum = 0;
- zeroBuckets = 0;
-
- byte[] registers = hllc.registers;
- for (int i = 0; i < hllc.m; i++) {
- if (registers[i] == 0) {
- registerSum++;
- zeroBuckets++;
- } else {
- registerSum += 1.0 / (1 << registers[i]);
- }
- }
- }
-
- public long getCountEstimate() {
- int m = (int) Math.pow(2, p);
- double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m));
- double alphaMM = alpha * m * m;
- double estimate = alphaMM / registerSum;
-
- // small cardinality adjustment
- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
- estimate = m * Math.log(m * 1.0 / zeroBuckets);
- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
- }
-
- return Math.round(estimate);
- }
- }
-
- // ============================================================================
-
- public void writeRegisters(final ByteBuffer out) throws IOException {
-
- final int indexLen = getRegisterIndexSize();
- int size = size();
-
- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
- byte scheme;
- if (5 + (indexLen + 1) * size < m) // 5 is max len of vint
- scheme = 0; // map
- else
- scheme = 1; // array
- out.put(scheme);
-
- if (scheme == 0) { // map scheme
- BytesUtil.writeVInt(size, out);
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0) {
- writeUnsigned(i, indexLen, out);
- out.put(registers[i]);
- }
- }
- } else if (scheme == 1) { // array scheme
- out.put(registers);
- } else
- throw new IllegalStateException();
- }
-
- public void readRegisters(ByteBuffer in) throws IOException {
- byte scheme = in.get();
-
- if (scheme == 0) { // map scheme
- clear();
- int size = BytesUtil.readVInt(in);
- if (size > m)
- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
- int indexLen = getRegisterIndexSize();
- for (int i = 0; i < size; i++) {
- int key = readUnsigned(in, indexLen);
- registers[key] = in.get();
- }
- } else if (scheme == 1) { // array scheme
- in.get(registers);
- } else
- throw new IllegalStateException();
- }
-
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
-
- byte scheme = in.get();
- if (scheme == 0) { // map scheme
- int size = BytesUtil.readVInt(in);
- int indexLen = getRegisterIndexSize();
- len = in.position() - mark + (indexLen + 1) * size;
- } else {
- len = in.position() - mark + m;
- }
-
- in.position(mark);
- return len;
- }
-
- public int maxLength() {
- return 1 + m;
- }
-
- public void writeRegistersArray(final ByteBuffer out) {
- out.put(this.registers);
- }
-
- public void readRegistersArray(ByteBuffer in) {
- in.get(registers, 0, m);
- }
-
- private int getRegisterIndexSize() {
- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
- result = prime * result + p;
- result = prime * result + Arrays.hashCode(registers);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
- if (hashFunc == null) {
- if (other.hashFunc != null)
- return false;
- } else if (!hashFunc.equals(other.hashFunc))
- return false;
- if (p != other.p)
- return false;
- if (!Arrays.equals(registers, other.registers))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(HyperLogLogPlusCounter o) {
- if (o == null)
- return 1;
-
- long e1 = this.getCountEstimate();
- long e2 = o.getCountEstimate();
-
- if (e1 == e2)
- return 0;
- else if (e1 > e2)
- return 1;
- else
- return -1;
- }
-
- public static void main(String[] args) throws IOException {
- dumpErrorRates();
- }
-
- static void dumpErrorRates() {
- for (int p = 10; p <= 18; p++) {
- double rate = new HyperLogLogPlusCounter(p).getErrorRate();
- double er = Math.round(rate * 10000) / 100D;
- double er2 = Math.round(rate * 2 * 10000) / 100D;
- double er3 = Math.round(rate * 3 * 10000) / 100D;
- long size = Math.round(Math.pow(2, p));
- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
- }
- }
-
- /**
- *
- * @param num
- * @param size
- * @param out
- */
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-}
[04/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
deleted file mode 100644
index fa7e51b..0000000
--- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ /dev/null
@@ -1,646 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by hongbin on 5/20/14.
- */
-public class FactTableGenerator {
- CubeInstance cube = null;
- CubeDesc desc = null;
- ResourceStore store = null;
- String factTableName = null;
-
- GenConfig genConf = null;
-
- Random r = null;
-
- String cubeName;
- long randomSeed;
- int rowCount;
- int unlinkableRowCount;
- int unlinkableRowCountMax;
- double conflictRatio;
- double linkableRatio;
-
- // the names of lookup table columns which is in relation with fact
- // table(appear as fk in fact table)
- TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // possible values of lookupTableKeys, extracted from existing lookup
- // tables.
- // The key is in the format of tablename/columnname
- TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // lookup table name -> sets of all composite keys
- TreeMap<String, HashSet<Array<String>>> lookupTableCompositeKeyValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- private void init(String cubeName, int rowCount, double conflictRaio, double linkableRatio, long randomSeed) {
- this.rowCount = rowCount;
- this.conflictRatio = conflictRaio;
- this.cubeName = cubeName;
- this.randomSeed = randomSeed;
- this.linkableRatio = linkableRatio;
-
- this.unlinkableRowCountMax = (int) (this.rowCount * (1 - linkableRatio));
- this.unlinkableRowCount = 0;
-
- r = new Random(randomSeed);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- cube = CubeManager.getInstance(config).getCube(cubeName);
- desc = cube.getDescriptor();
- factTableName = desc.getFactTable();
- store = ResourceStore.getStore(config);
- }
-
- /*
- * users can specify the value preference for each column
- */
- private void loadConfig() {
- try {
- InputStream configStream = store.getResource("/data/data_gen_config.json").inputStream;
- this.genConf = GenConfig.loadConfig(configStream);
-
- if (configStream != null)
- configStream.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void loadLookupTableValues(String lookupTableName, LinkedList<String> columnNames, int distinctRowCount) throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- // only deal with composite keys
- if (columnNames.size() > 1 && !lookupTableCompositeKeyValues.containsKey(lookupTableName)) {
- lookupTableCompositeKeyValues.put(lookupTableName, new HashSet<Array<String>>());
- }
-
- InputStream tableStream = null;
- BufferedReader tableReader = null;
- try {
- TreeMap<String, Integer> zeroBasedInice = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- for (String columnName : columnNames) {
- ColumnDesc cDesc = MetadataManager.getInstance(config).getTableDesc(lookupTableName).findColumnByName(columnName);
- zeroBasedInice.put(columnName, cDesc.getZeroBasedIndex());
- }
-
- String path = "/data/" + lookupTableName + ".csv";
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
- tableReader.mark(0);
- int rowCount = 0;
- int curRowNum = 0;
- String curRow;
-
- while (tableReader.readLine() != null)
- rowCount++;
-
- HashSet<Integer> rows = new HashSet<Integer>();
- distinctRowCount = (distinctRowCount < rowCount) ? distinctRowCount : rowCount;
- while (rows.size() < distinctRowCount) {
- rows.add(r.nextInt(rowCount));
- }
-
- // reopen the stream
- tableReader.close();
- tableStream.close();
- tableStream = null;
- tableReader = null;
-
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
-
- while ((curRow = tableReader.readLine()) != null) {
- if (rows.contains(curRowNum)) {
- String[] tokens = curRow.split(",");
-
- String[] comboKeys = null;
- int index = 0;
- if (columnNames.size() > 1)
- comboKeys = new String[columnNames.size()];
-
- for (String columnName : columnNames) {
- int zeroBasedIndex = zeroBasedInice.get(columnName);
- if (!feasibleValues.containsKey(lookupTableName + "/" + columnName))
- feasibleValues.put(lookupTableName + "/" + columnName, new ArrayList<String>());
- feasibleValues.get(lookupTableName + "/" + columnName).add(tokens[zeroBasedIndex]);
-
- if (columnNames.size() > 1) {
- comboKeys[index] = tokens[zeroBasedIndex];
- index++;
- }
- }
-
- if (columnNames.size() > 1) {
- Array<String> wrap = new Array<String>(comboKeys);
- if (lookupTableCompositeKeyValues.get(lookupTableName).contains(wrap)) {
- throw new Exception("The composite key already exist in the lookup table");
- }
- lookupTableCompositeKeyValues.get(lookupTableName).add(wrap);
- }
- }
- curRowNum++;
- }
-
- if (tableStream != null)
- tableStream.close();
- if (tableReader != null)
- tableReader.close();
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- // prepare the candidate values for each joined column
- private void prepare() throws Exception {
- // load config
- loadConfig();
-
- TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- for (DimensionDesc dim : desc.getDimensions()) {
- for (TblColRef col : dim.getColumnRefs()) {
- if (col.getTable().equals(factTableName))
- factTableColumns.add(col.getName());
- }
-
- JoinDesc join = dim.getJoin();
- if (join != null) {
- String lookupTable = dim.getTable();
- for (String column : join.getPrimaryKey()) {
- if (!lookupTableKeys.containsKey(lookupTable)) {
- lookupTableKeys.put(lookupTable, new LinkedList<String>());
- }
-
- if (!lookupTableKeys.get(lookupTable).contains(column))
- lookupTableKeys.get(lookupTable).add(column);
- }
- }
- }
-
- int distinctRowCount = (int) (this.rowCount / this.conflictRatio);
- distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount;
- // lookup tables
- for (String lookupTable : lookupTableKeys.keySet()) {
- this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount);
- }
- }
-
- private List<DimensionDesc> getSortedDimentsionDescs() {
- List<DimensionDesc> dimensions = desc.getDimensions();
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc o1, DimensionDesc o2) {
- JoinDesc j1 = o2.getJoin();
- JoinDesc j2 = o1.getJoin();
- return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0);
- }
- });
- return dimensions;
- }
-
- /**
- * Generate the fact table and return it as text
- *
- * @return
- * @throws Exception
- */
- private String cookData() throws Exception {
- // the columns on the fact table can be classified into three groups:
- // 1. foreign keys
- TreeMap<String, String> factTableCol2LookupCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- // 2. metrics or directly used dimensions
- TreeSet<String> usedCols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- // 3. others, not referenced anywhere
-
- TreeMap<String, String> lookupCol2factTableCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // find fact table columns in fks
- List<DimensionDesc> dimensions = getSortedDimentsionDescs();
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc != null) {
- String[] fks = jDesc.getForeignKey();
- String[] pks = jDesc.getPrimaryKey();
- int num = fks.length;
- for (int i = 0; i < num; ++i) {
- String value = dim.getTable() + "/" + pks[i];
-
- lookupCol2factTableCol.put(value, fks[i]);
-
- if (factTableCol2LookupCol.containsKey(fks[i])) {
- if (!factTableCol2LookupCol.get(fks[i]).equals(value)) {
- System.out.println("Warning: Disambiguation on the mapping of column " + fks[i] + ", " + factTableCol2LookupCol.get(fks[i]) + "(chosen) or " + value);
- continue;
- }
- }
- factTableCol2LookupCol.put(fks[i], value);
- }
- }
- //else, deal with it in next roung
- }
-
- // find fact table columns in direct dimension
- // DO NOT merge this with the previous loop
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc == null) {
- // column on fact table used directly as a dimension
- for (String aColumn : dim.getColumn()) {
- if (!factTableCol2LookupCol.containsKey(aColumn))
- usedCols.add(aColumn);
- }
- }
- }
-
- // find fact table columns in measures
- for (MeasureDesc mDesc : desc.getMeasures()) {
- List<TblColRef> pcols = mDesc.getFunction().getParameter().getColRefs();
- if (pcols != null) {
- for (TblColRef col : pcols) {
- if (!factTableCol2LookupCol.containsKey(col.getName()))
- usedCols.add(col.getName());
- }
- }
- }
-
- return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols);
- }
-
- private String normToTwoDigits(int v) {
- if (v < 10)
- return "0" + v;
- else
- return Integer.toString(v);
- }
-
- private String randomPick(ArrayList<String> candidates) {
- int index = r.nextInt(candidates.size());
- return candidates.get(index);
- }
-
- private String createRandomCell(ColumnDesc cDesc, ArrayList<String> range) throws Exception {
- DataType type = cDesc.getType();
- if (type.isStringFamily()) {
- throw new Exception("Can't handle range values for string");
-
- } else if (type.isIntegerFamily()) {
- int low = Integer.parseInt(range.get(0));
- int high = Integer.parseInt(range.get(1));
- return Integer.toString(r.nextInt(high - low) + low);
-
- } else if (type.isDouble()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isFloat()) {
- float low = Float.parseFloat(range.get(0));
- float high = Float.parseFloat(range.get(1));
- return String.format("%.4f", r.nextFloat() * (high - low) + low);
-
- } else if (type.isDecimal()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isDateTimeFamily()) {
-
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
- Date start = format.parse(range.get(0));
- Date end = format.parse(range.get(1));
- long diff = end.getTime() - start.getTime();
- Date temp = new Date(start.getTime() + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createRandomCell(ColumnDesc cDesc) {
- String type = cDesc.getTypeName();
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 2; i++) {
- sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
- // possible strings
- }
- return sb.toString();
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return Integer.toString(r.nextInt(128));
- } else if (s.equals("double")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("float")) {
- return String.format("%.4f", r.nextFloat() * 100);
- } else if (s.equals("decimal")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("date")) {
- long date20131231 = 61349312153265L;
- long date20010101 = 60939158400000L;
- long diff = date20131231 - date20010101;
- Date temp = new Date(date20010101 + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createDefaultsCell(String type) {
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
- return "abcde";
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return "0";
- } else if (s.equals("double")) {
- return "0";
- } else if (s.equals("float")) {
- return "0";
- } else if (s.equals("decimal")) {
- return "0";
- } else if (s.equals("date")) {
- return "1970-01-01";
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private void printColumnMappings(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) {
-
- System.out.println("=======================================================================");
- System.out.format("%-30s %s", "FACT_TABLE_COLUMN", "MAPPING");
- System.out.println();
- System.out.println();
- for (Map.Entry<String, String> entry : factTableCol2LookupCol.entrySet()) {
- System.out.format("%-30s %s", entry.getKey(), entry.getValue());
- System.out.println();
- }
- for (String key : usedCols) {
- System.out.format("%-30s %s", key, "Random Values");
- System.out.println();
- }
- for (String key : defaultColumns) {
- System.out.format("%-30s %s", key, "Default Values");
- System.out.println();
- }
- System.out.println("=======================================================================");
-
- System.out.println("Parameters:");
- System.out.println();
- System.out.println("CubeName: " + cubeName);
- System.out.println("RowCount: " + rowCount);
- System.out.println("ConflictRatio: " + conflictRatio);
- System.out.println("LinkableRatio: " + linkableRatio);
- System.out.println("Seed: " + randomSeed);
- System.out.println();
- System.out.println("The number of actual unlinkable fact rows is: " + this.unlinkableRowCount);
- System.out.println("You can vary the above parameters to generate different datasets.");
- System.out.println();
- }
-
- // Any row in the column must finally appear in the flatten big table.
- // for single-column joins the generated row is guaranteed to have a match
- // in lookup table
- // for composite keys we'll need an extra check
- private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- for (String lookupTable : lookupTableKeys.keySet()) {
- if (lookupTableKeys.get(lookupTable).size() == 1)
- continue;
-
- String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()];
- int index = 0;
- for (String column : lookupTableKeys.get(lookupTable)) {
- String key = lookupTable + "/" + column;
- String factTableCol = lookupCol2FactTableCol.get(key);
- int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex();
- comboKey[index] = columnValues.get(cardinal);
-
- index++;
- }
- Array<String> wrap = new Array<String>(comboKey);
- if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) {
- // System.out.println("Try " + wrap + " Failed, continue...");
- return false;
- }
- }
- return true;
- }
-
- private String createCell(ColumnDesc cDesc) throws Exception {
- ColumnConfig cConfig = null;
-
- if ((cConfig = genConf.getColumnConfigByName(cDesc.getName())) == null) {
- // if the column is not configured, use random values
- return (createRandomCell(cDesc));
-
- } else {
- // the column has a configuration
- if (!cConfig.isAsRange() && !cConfig.isExclusive() && r.nextBoolean()) {
- // if the column still allows random values
- return (createRandomCell(cDesc));
-
- } else {
- // use specified values
- ArrayList<String> valueSet = cConfig.getValueSet();
- if (valueSet == null || valueSet.size() == 0)
- throw new Exception("Did you forget to specify value set for " + cDesc.getName());
-
- if (!cConfig.isAsRange()) {
- return (randomPick(valueSet));
- } else {
- if (valueSet.size() != 2)
- throw new Exception("Only two values can be set for range values, the column: " + cDesc.getName());
-
- return (createRandomCell(cDesc, valueSet));
- }
- }
-
- }
- }
-
- private LinkedList<String> createRow(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- LinkedList<String> columnValues = new LinkedList<String>();
-
- for (ColumnDesc cDesc : MetadataManager.getInstance(config).getTableDesc(factTableName).getColumns()) {
-
- String colName = cDesc.getName();
-
- if (factTableCol2LookupCol.containsKey(colName)) {
-
- // if the current column is a fk column in fact table
- ArrayList<String> candidates = this.feasibleValues.get(factTableCol2LookupCol.get(colName));
-
- columnValues.add(candidates.get(r.nextInt(candidates.size())));
- } else if (usedCols.contains(colName)) {
-
- // if the current column is a metric column in fact table
- columnValues.add(createCell(cDesc));
- } else {
-
- // otherwise this column is not useful in OLAP
- columnValues.add(createDefaultsCell(cDesc.getTypeName()));
- defaultColumns.add(colName);
- }
- }
-
- return columnValues;
- }
-
- /**
- * return the text of table contents(one line one row)
- *
- * @param rowCount
- * @param factTableCol2LookupCol
- * @param lookupCol2FactTableCol
- * @param usedCols
- * @return
- * @throws Exception
- */
- private String createTable(int rowCount, TreeMap<String, String> factTableCol2LookupCol, TreeMap<String, String> lookupCol2FactTableCol, TreeSet<String> usedCols) throws Exception {
- try {
- TreeSet<String> defaultColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < rowCount;) {
-
- LinkedList<String> columnValues = createRow(factTableCol2LookupCol, usedCols, defaultColumns);
-
- if (!matchAllCompositeKeys(lookupCol2FactTableCol, columnValues)) {
- if (unlinkableRowCount < unlinkableRowCountMax) {
- unlinkableRowCount++;
- } else {
- continue;
- }
- }
-
- for (String c : columnValues)
- sb.append(c + ",");
- sb.deleteCharAt(sb.length() - 1);
- sb.append(System.getProperty("line.separator"));
-
- i++;
-
- // System.out.println("Just generated the " + i + "th record");
- }
-
- printColumnMappings(factTableCol2LookupCol, usedCols, defaultColumns);
-
- return sb.toString();
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- return null;
- }
-
- /**
- * Randomly create a fact table and return the table content
- *
- * @param cubeName name of the cube
- * @param rowCount expected row count generated
- * @param linkableRatio the percentage of fact table rows that can be linked with all
- * lookup table by INNER join
- * @param randomSeed random seed
- */
- public static String generate(String cubeName, String rowCount, String linkableRatio, String randomSeed, String joinType) throws Exception {
-
- if (cubeName == null)
- cubeName = "test_kylin_cube_with_slr_ready";
- if (rowCount == null)
- rowCount = "10000";
- if (linkableRatio == null)
- linkableRatio = "0.6";
-
- //if (randomSeed == null)
- // don't give it value
-
- // String conflictRatio = "5";//this parameter do not allow configuring
- // any more
-
- FactTableGenerator generator = new FactTableGenerator();
- long seed;
- if (randomSeed != null) {
- seed = Long.parseLong(randomSeed);
- } else {
- Random r = new Random();
- seed = r.nextLong();
- }
-
- generator.init(cubeName, Integer.parseInt(rowCount), 5, Double.parseDouble(linkableRatio), seed);
- generator.prepare();
- return generator.cookData();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
deleted file mode 100644
index f7d7341..0000000
--- a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.JsonUtil;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-
-/**
- * Created by honma on 5/29/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class GenConfig {
-
- @JsonProperty("columnConfigs")
- private ArrayList<ColumnConfig> columnConfigs;
-
- private HashMap<String, ColumnConfig> cache = new HashMap<String, ColumnConfig>();
-
- public ArrayList<ColumnConfig> getColumnConfigs() {
- return columnConfigs;
- }
-
- public void setColumnConfigs(ArrayList<ColumnConfig> columnConfigs) {
- this.columnConfigs = columnConfigs;
- }
-
- public ColumnConfig getColumnConfigByName(String columnName) {
- columnName = columnName.toLowerCase();
-
- if (cache.containsKey(columnName))
- return cache.get(columnName);
-
- for (ColumnConfig cConfig : columnConfigs) {
- if (cConfig.getColumnName().toLowerCase().equals(columnName)) {
- cache.put(columnName, cConfig);
- return cConfig;
- }
- }
- cache.put(columnName, null);
- return null;
- }
-
- public static GenConfig loadConfig(InputStream stream) {
- try {
- GenConfig config = JsonUtil.readValue(stream, GenConfig.class);
- return config;
- } catch (JsonMappingException e) {
- e.printStackTrace();
- } catch (JsonParseException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
deleted file mode 100644
index 8d22efb..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
-
- public static void setLocalMR(Configuration conf) {
- conf.set("fs.default.name", "file:///");
- conf.set("mapreduce.framework.name", "local");
- conf.set("mapreduce.application.framework.path", "");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
- }
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/flat_table/";
- String output = "target/test-output/base_cuboid/";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "base_cuboid_job";
- String level = "0";
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
deleted file mode 100644
index f87bb0a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class BaseCuboidMapperPerformanceTest {
-
- String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure";
- String cubeName = "test_kylin_cube_with_slr";
- Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq");
-
- @Ignore("convenient trial tool for dev")
- @Test
- public void test() throws IOException, InterruptedException {
- Configuration hconf = new Configuration();
- BaseCuboidMapper mapper = new BaseCuboidMapper();
- Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
-
- mapper.setup(context);
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- while (reader.next(key, value)) {
- mapper.map(key, value, context);
- }
-
- reader.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
deleted file mode 100644
index 58d093a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowKeyDecoder;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
-
- MapDriver<Text, Text, Text, Text> mapDriver;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapperWithHeader() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.3322"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", "1", "22");
- }
-
- private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String... valueStr) {
- MeasureCodec codec = new MeasureCodec(measures);
- Object[] values = new Object[measures.size()];
- codec.decode(ByteBuffer.wrap(valueBytes.getBytes()), values);
- assertTrue(new BigDecimal(valueStr[0]).equals(values[0]));
- assertTrue(new BigDecimal(valueStr[1]).equals(values[1]));
- assertTrue(new BigDecimal(valueStr[2]).equals(values[2]));
- assertTrue(new LongWritable(Long.valueOf(valueStr[3])).equals(values[3]));
- assertTrue(new LongWritable(Long.valueOf(valueStr[4])).equals(values[4]));
- }
-
- @Test
- public void testMapperWithNull() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N22"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", "1", "22");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
deleted file mode 100644
index a8e29e6..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * @author yangli9
- *
- */
-public class CopySeq {
-
- public static void main(String[] args) throws IOException {
- copyTo64MB(args[0], args[1]);
- }
-
- public static void copyTo64MB(String src, String dst) throws IOException {
- Configuration hconf = new Configuration();
- Path srcPath = new Path(src);
- Path dstPath = new Path(dst);
-
- FileSystem fs = FileSystem.get(hconf);
- long srcSize = fs.getFileStatus(srcPath).getLen();
- int copyTimes = (int) (67108864 / srcSize); // 64 MB
- System.out.println("Copy " + copyTimes + " times");
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf)));
-
- int count = 0;
- while (reader.next(key, value)) {
- for (int i = 0; i < copyTimes; i++) {
- writer.append(key, value);
- count++;
- }
- }
-
- System.out.println("Len: " + writer.getLength());
- System.out.println("Rows: " + count);
-
- reader.close();
- writer.close();
- }
-
- static CompressionCodec getLZOCodec(Configuration hconf) {
- CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
- return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
deleted file mode 100644
index c9b7eba..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.measure.MeasureCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
-
- String cubeName = "test_kylin_cube_with_slr_ready";
-
- MeasureCodec codec;
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- Object[] outKV = new Object[2];
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
- CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor();
- codec = new MeasureCodec(desc.getMeasures());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testBasic() throws Exception {
-
- Configuration hconf = new Configuration();
- Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV);
-
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapper.setup(context);
-
- Text key = new Text("not important");
- Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
-
- mapper.map(key, value, context);
-
- ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
- KeyValue outValue = (KeyValue) outKV[1];
-
- assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0);
-
- assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
deleted file mode 100644
index 3555782..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeHFileMapperTest {
-
- MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver;
-
- private String cube_name = "FLAT_ITEM_CUBE";
-
- @Before
- public void setUp() {
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- @Ignore("not maintaining")
- public void testMapper2() throws IOException {
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);
-
- mapDriver.addInput(new Text("52010tech"), new Text("35.432"));
-
- List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run();
-
- assertEquals(2, result.size());
-
- byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
- ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);
-
- Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0);
- Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
-
- assertEquals(key, p1.getFirst());
- assertEquals("cf1", new String(p1.getSecond().getFamily()));
- assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
- assertEquals("35.43", new String(p1.getSecond().getValue()));
-
- assertEquals(key, p2.getFirst());
- assertEquals("cf1", new String(p2.getSecond().getFamily()));
- assertEquals("item_count", new String(p2.getSecond().getQualifier()));
- assertEquals("2", new String(p2.getSecond().getValue()));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
deleted file mode 100644
index 51f3990..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CubeReducerTest extends LocalFileMetadataTestCase {
-
- ReduceDriver<Text, Text, Text, Text> reduceDriver;
-
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- CuboidReducer reducer = new CuboidReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testReducer() throws Exception {
-
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
-
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 22));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 23));
- values1.add(newValueText(codec, "10", "10", "10", 1, 24));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 12));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 13));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4, 11));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 69));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 25));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4, 11));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- @Test
- public void testReducerOnlyAggrInBaseCuboid() throws Exception {
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
- reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);
-
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
- FunctionDesc functionDesc = measureDesc.getFunction();
- Field field = FunctionDesc.class.getDeclaredField("measureType");
- field.setAccessible(true);
- MeasureType origMeasureType = functionDesc.getMeasureType();
- field.set(functionDesc, new MockUpMeasureType(origMeasureType));
-
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
- values1.add(newValueText(codec, "10", "10", "10", 1, 300));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count, int itemcount) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count), new LongWritable(itemcount) };
-
- buf.clear();
- codec.encode(values, buf);
-
- Text t = new Text();
- t.set(buf.array(), 0, buf.position());
- return t;
- }
-
- class MockUpMeasureType extends MeasureType {
- MeasureType origMeasureType;
-
- public MockUpMeasureType(MeasureType origMeasureType) {
- this.origMeasureType = origMeasureType;
- }
-
- @Override
- public boolean onlyAggrInBaseCuboid() {
- return true;
- }
-
- @Override
- public MeasureIngester newIngester() {
- return origMeasureType.newIngester();
- }
-
- @Override
- public MeasureAggregator newAggregator() {
- return origMeasureType.newAggregator();
- }
-
- @Override
- public boolean needRewrite() {
- return origMeasureType.needRewrite();
- }
-
- @Override
- public Class<?> getRewriteCalciteAggrFunctionClass() {
- return origMeasureType.getRewriteCalciteAggrFunctionClass();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
deleted file mode 100644
index 6f7db6a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileUtil;
-//import org.apache.hadoop.util.ToolRunner;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionJobTest {
-//
-// private Configuration conf;
-//
-// @Before
-// public void setup() throws IOException {
-// conf = new Configuration();
-// BaseCuboidJobTest.setLocalMR(conf);
-// }
-//
-// @Test
-// public void testJob() throws Exception {
-// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
-// final String output = "target/test-output/key_distribution/";
-// final String cubeName = "test_kylin_cube_with_slr";
-// final String metadata = MetadataManager.getMetadataUrlFromEnv();
-//
-// FileUtil.fullyDelete(new File(output));
-//
-// String[] args =
-// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata,
-// "-columnpercentage", "50", "-splitnumber", "10" };
-// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args));
-// }
-//
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
deleted file mode 100644
index b4577f7..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.util.List;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-//import org.apache.hadoop.mrunit.types.Pair;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.index.BatchConstants;
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author ysong1
-// *
-// */
-//public class KeyDistributionMapperTest {
-// @SuppressWarnings("rawtypes")
-// MapDriver mapDriver;
-// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-//
-// @Before
-// public void setUp() {
-// KeyDistributionMapper mapper = new KeyDistributionMapper();
-// mapDriver = MapDriver.newMapDriver(mapper);
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithoutHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2,
-// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3,
-// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4,
-// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5,
-// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6,
-// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
deleted file mode 100644
index 6c17c69..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- // String input =
- // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid";
- String output = "target/test-output/merged_cuboid";
- String cubeName = "test_kylin_cube_with_slr_ready";
- String jobname = "merge_cuboid";
-
- File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- FileUtils.forceDelete(baseFolder);
- baseFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- FileUtils.forceDeleteOnExit(baseFolder);
-
- File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d");
- FileUtils.forceDelete(eightFolder);
- eightFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder);
- FileUtils.forceDeleteOnExit(eightFolder);
-
- FileUtil.fullyDelete(new File(output));
-
- // CubeManager cubeManager =
- // CubeManager.getInstanceFromEnv(getTestConfig());
-
- String[] args = { "-input", baseFolder.getAbsolutePath() + "," + eightFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
deleted file mode 100644
index ea5c163..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.ListDictionaryValueEnumerator;
-import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-@SuppressWarnings("rawtypes")
-public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
-
- private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class);
-
- MapDriver<Text, Text, Text, Text> mapDriver;
- CubeManager cubeManager;
- CubeInstance cube;
- DictionaryManager dictionaryManager;
-
- TblColRef lfn;
- TblColRef lsi;
- TblColRef ssc;
-
- private DictionaryInfo makeSharedDict() throws IOException {
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_common_dict");
-
- DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 101, 101, 101 });
- values.add(new byte[] { 102, 102, 102 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values));
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- return newDictInfo;
- }
-
- @Before
- public void setUp() throws Exception {
-
- createTestMetadata();
-
- logger.info("The metadataUrl is : " + getTestConfig());
-
- MetadataManager.clearCache();
- CubeManager.clearCache();
- ProjectManager.clearCache();
- DictionaryManager.clearCache();
-
- // hack for distributed cache
- // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
- // make sure the following mapper could get latest CubeManger
- FileUtils.deleteDirectory(new File("../job/meta"));
-
- MergeCuboidMapper mapper = new MergeCuboidMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
-
- cubeManager = CubeManager.getInstance(getTestConfig());
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
- dictionaryManager = DictionaryManager.getInstance(getTestConfig());
- lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
- lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
- ssc = cube.getDescriptor().findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
-
- DictionaryInfo sharedDict = makeSharedDict();
-
- boolean isFirstSegment = true;
- for (CubeSegment segment : cube.getSegments()) {
-
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
-
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 97, 97, 97 });
- if (isFirstSegment)
- values.add(new byte[] { 99, 99, 99 });
- else
- values.add(new byte[] { 98, 98, 98 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values));
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- segment.putDictResPath(lfn, newDictInfo.getResourcePath());
- segment.putDictResPath(lsi, sharedDict.getResourcePath());
- segment.putDictResPath(ssc, sharedDict.getResourcePath());
-
- // cubeManager.saveResource(segment.getCubeInstance());
- // cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
-
- isFirstSegment = false;
- }
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void test() throws IOException, ParseException {
-
- String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";
-
- CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, 1386835200000L, false);
- String segmentName = newSeg.getName();
-
- final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn);
- assertTrue(dictionary == null);
- // ((TrieDictionary) dictionary).dump(System.out);
-
- // hack for distributed cache
- // File metaDir = new File("../job/meta");
- // FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), metaDir);
- //
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
- // // "../job/meta");
- //
- // byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
- // byte[] value = new byte[] { 1, 2, 3 };
- // byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
- // byte[] newvalue = new byte[] { 1, 2, 3 };
- //
- // mapDriver.withInput(new Text(key), new Text(value));
- // mapDriver.withOutput(new Text(newkey), new Text(newvalue));
- // mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
- //
- // mapDriver.runTest();
- }
-}
[07/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
deleted file mode 100644
index b1a17e7..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexJob extends AbstractHadoopJob {
- protected static final Logger log = LoggerFactory.getLogger(InvertedIndexJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_TABLE_NAME);
- options.addOption(OPTION_OUTPUT_PATH);
- parseOptions(options, args);
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- String iiname = getOptionValue(OPTION_II_NAME);
- String intermediateTable = getOptionValue(OPTION_TABLE_NAME);
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- // ----------------------------------------------------------------------------
-
- System.out.println("Starting: " + job.getJobName());
-
- IIInstance ii = getII(iiname);
- short sharding = ii.getDescriptor().getSharding();
-
- setJobClasspath(job);
-
- setupMapper(intermediateTable);
- setupReducer(output, sharding);
- attachMetadata(ii);
-
- return waitForCompletion(job);
-
- } catch (Exception e) {
- printUsage(options);
- throw e;
- } finally {
- if (job != null)
- cleanupTempConfFile(job.getConfiguration());
- }
- }
-
- private IIInstance getII(String iiName) {
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = mgr.getII(iiName);
- if (ii == null)
- throw new IllegalArgumentException("No Inverted Index found by name " + iiName);
- return ii;
- }
-
- private void attachMetadata(IIInstance ii) throws IOException {
-
- Configuration conf = job.getConfiguration();
- attachKylinPropsAndMetadata(ii, conf);
-
- IISegment seg = ii.getFirstSegment();
- conf.set(BatchConstants.CFG_II_NAME, ii.getName());
- conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName());
- }
-
- private void setupMapper(String intermediateTable) throws IOException {
-
- String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
- HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);
-
- job.setInputFormatClass(HCatInputFormat.class);
-
- job.setMapperClass(InvertedIndexMapper.class);
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(ImmutableBytesWritable.class);
- job.setPartitionerClass(InvertedIndexPartitioner.class);
- }
-
- private void setupReducer(Path output, short sharding) throws IOException {
- job.setReducerClass(InvertedIndexReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(ImmutableBytesWritable.class);
- job.setOutputValueClass(ImmutableBytesWritable.class);
-
- job.setNumReduceTasks(sharding);
-
- FileOutputFormat.setOutputPath(job, output);
-
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
-
- deletePath(job.getConfiguration(), output);
- }
-
- public static void main(String[] args) throws Exception {
- InvertedIndexJob job = new InvertedIndexJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexMapper.java
deleted file mode 100644
index b3baafe..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexMapper.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, LongWritable, ImmutableBytesWritable> {
-
- private TableRecordInfo info;
- private TableRecord rec;
-
- private LongWritable outputKey;
- private ImmutableBytesWritable outputValue;
- private HCatSchema schema = null;
- private List<HCatFieldSchema> fields;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
- this.info = new TableRecordInfo(seg);
- this.rec = this.info.createTableRecord();
-
- outputKey = new LongWritable();
- outputValue = new ImmutableBytesWritable(rec.getBytes());
-
- schema = HCatInputFormat.getTableSchema(context.getConfiguration());
-
- fields = schema.getFields();
- }
-
- @Override
- public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {
-
- rec.reset();
- for (int i = 0; i < fields.size(); i++) {
- Object fieldValue = record.get(i);
- rec.setValueString(i, fieldValue == null ? null : fieldValue.toString());
- }
-
- outputKey.set(rec.getTimestamp());
- // outputValue's backing bytes array is the same as rec
-
- context.write(outputKey, outputValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexPartitioner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexPartitioner.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexPartitioner.java
deleted file mode 100644
index 113d4ed..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexPartitioner.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexPartitioner extends Partitioner<LongWritable, ImmutableBytesWritable> implements Configurable {
-
- private Configuration conf;
- private TableRecordInfo info;
- private TableRecord rec;
-
- @Override
- public int getPartition(LongWritable key, ImmutableBytesWritable value, int numPartitions) {
- rec.setBytes(value.get(), value.getOffset(), value.getLength());
- return rec.getShard();
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- try {
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
- this.info = new TableRecordInfo(seg);
- this.rec = this.info.createTableRecord();
- } catch (IOException e) {
- throw new RuntimeException("", e);
- }
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
deleted file mode 100644
index 37b02b8..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.SliceBuilder;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable> {
-
- private TableRecordInfo info;
- private TableRecord rec;
- private SliceBuilder builder;
- private IIKeyValueCodec kv;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
- info = new TableRecordInfo(seg);
- rec = info.createTableRecord();
- builder = null;
- kv = new IIKeyValueCodec(info.getDigest());
- }
-
- @Override
- public void reduce(LongWritable key, Iterable<ImmutableBytesWritable> values, Context context) //
- throws IOException, InterruptedException {
- for (ImmutableBytesWritable v : values) {
- rec.setBytes(v.get(), v.getOffset(), v.getLength());
-
- if (builder == null) {
- builder = new SliceBuilder(info, rec.getShard());
- }
-
- //TODO: to delete this log
- System.out.println(rec.getShard() + " - " + rec);
-
- Slice slice = builder.append(rec);
- if (slice != null) {
- output(slice, context);
- }
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- Slice slice = builder.close();
- if (slice != null) {
- output(slice, context);
- }
- }
-
- private void output(Slice slice, Context context) throws IOException, InterruptedException {
- for (Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair : kv.encodeKeyValue(slice)) {
- context.write(pair.getFirst(), pair.getSecond());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
deleted file mode 100644
index 8ac5650..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author ysong1
- *
- */
-@SuppressWarnings("static-access")
-public class RandomKeyDistributionJob extends AbstractHadoopJob {
-
- protected static final Logger log = LoggerFactory.getLogger(RandomKeyDistributionJob.class);
-
- static final Option OPTION_KEY_CLASS = OptionBuilder.withArgName("keyclass").hasArg().isRequired(true).withDescription("Key Class").create("keyclass");
- static final Option OPTION_REGION_MB = OptionBuilder.withArgName("regionmb").hasArg().isRequired(true).withDescription("MB per Region").create("regionmb");
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_KEY_CLASS);
- options.addOption(OPTION_REGION_MB);
-
- parseOptions(options, args);
-
- // start job
- String jobName = getOptionValue(OPTION_JOB_NAME);
- job = Job.getInstance(getConf(), jobName);
-
- setJobClasspath(job);
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- FileOutputFormat.setOutputPath(job, output);
-
- String keyClass = getOptionValue(OPTION_KEY_CLASS);
- Class<?> keyClz = Class.forName(keyClass);
-
- int regionMB = Integer.parseInt(getOptionValue(OPTION_REGION_MB));
-
- // Mapper
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(RandomKeyDistributionMapper.class);
- job.setMapOutputKeyClass(keyClz);
- job.setMapOutputValueClass(NullWritable.class);
-
- // Reducer - only one
- job.setReducerClass(RandomKeyDistributionReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(keyClz);
- job.setOutputValueClass(NullWritable.class);
- job.setNumReduceTasks(1);
-
- this.deletePath(job.getConfiguration(), output);
-
- // total map input MB
- double totalMapInputMB = this.getTotalMapInputMB();
- int regionCount = Math.max(1, (int) (totalMapInputMB / regionMB));
- int mapSampleNumber = 1000;
- System.out.println("Total Map Input MB: " + totalMapInputMB);
- System.out.println("Region Count: " + regionCount);
-
- // set job configuration
- job.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, String.valueOf(mapSampleNumber));
- job.getConfiguration().set(BatchConstants.REGION_NUMBER, String.valueOf(regionCount));
-
- return waitForCompletion(job);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new RandomKeyDistributionJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionMapper.java
deleted file mode 100644
index 3914830..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionMapper.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.RandomSampler;
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionMapper<KEY extends Writable, VALUE> extends KylinMapper<KEY, VALUE, KEY, NullWritable> {
-
- private Configuration conf;
- private int sampleNumber;
- private List<KEY> allKeys;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- conf = context.getConfiguration();
- allKeys = new ArrayList<KEY>();
- sampleNumber = Integer.parseInt(conf.get(BatchConstants.MAPPER_SAMPLE_NUMBER));
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void map(KEY key, VALUE value, Context context) throws IOException, InterruptedException {
- KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf);
- ReflectionUtils.copy(conf, key, keyCopy);
- allKeys.add(keyCopy);
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- RandomSampler<KEY> sampler = new RandomSampler<KEY>();
- List<KEY> sampleResult = sampler.sample(allKeys, sampleNumber);
- for (KEY k : sampleResult) {
- context.write(k, NullWritable.get());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionReducer.java
deleted file mode 100644
index b96d18b..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionReducer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionReducer<KEY extends Writable> extends KylinReducer<KEY, NullWritable, KEY, NullWritable> {
-
- private Configuration conf;
- private int regionNumber;
- private List<KEY> allSplits;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- conf = context.getConfiguration();
- allSplits = new ArrayList<KEY>();
- regionNumber = Integer.parseInt(context.getConfiguration().get(BatchConstants.REGION_NUMBER));
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void reduce(KEY key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
- KEY keyCopy = (KEY) ReflectionUtils.newInstance(key.getClass(), conf);
- ReflectionUtils.copy(conf, key, keyCopy);
- allSplits.add(keyCopy);
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- int stepLength = allSplits.size() / regionNumber;
- for (int i = stepLength; i < allSplits.size(); i += stepLength) {
- context.write(allSplits.get(i), NullWritable.get());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
deleted file mode 100644
index 1cc20df..0000000
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultContext.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.impl.threadpool;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.execution.Executable;
-import org.apache.kylin.job.execution.ExecutableContext;
-
-/**
- * Created by qianzhou on 12/16/14.
- */
-public class DefaultContext implements ExecutableContext {
-
- private final ConcurrentMap<String, Executable> runningJobs;
- private final KylinConfig kylinConfig;
-
- public DefaultContext(ConcurrentMap<String, Executable> runningJobs, KylinConfig kylinConfig) {
- this.runningJobs = runningJobs;
- this.kylinConfig = kylinConfig;
- }
-
- @Override
- public Object getSchedulerContext() {
- return null;
- }
-
- @Override
- public KylinConfig getConfig() {
- return kylinConfig;
- }
-
- void addRunningJob(Executable executable) {
- runningJobs.put(executable.getId(), executable);
- }
-
- void removeRunningJob(Executable executable) {
- runningJobs.remove(executable.getId());
- }
-
- public Map<String, Executable> getRunningJobs() {
- return Collections.unmodifiableMap(runningJobs);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
deleted file mode 100644
index 46592f7..0000000
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.impl.threadpool;
-
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.locks.InterProcessMutex;
-import org.apache.curator.framework.state.ConnectionState;
-import org.apache.curator.framework.state.ConnectionStateListener;
-import org.apache.kylin.common.lock.JobLock;
-import org.apache.kylin.job.Scheduler;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.exception.SchedulerException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.Executable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public class DefaultScheduler implements Scheduler<AbstractExecutable>, ConnectionStateListener {
-
- private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
-
- private ExecutableManager executableManager;
- private FetcherRunner fetcher;
- private ScheduledExecutorService fetcherPool;
- private ExecutorService jobPool;
- private DefaultContext context;
-
- private Logger logger = LoggerFactory.getLogger(DefaultScheduler.class);
- private volatile boolean initialized = false;
- private volatile boolean hasStarted = false;
- private CuratorFramework zkClient;
- private JobEngineConfig jobEngineConfig;
- private InterProcessMutex sharedLock;
-
- private static final DefaultScheduler INSTANCE = new DefaultScheduler();
-
- private DefaultScheduler() {
- }
-
- private class FetcherRunner implements Runnable {
-
- @Override
- synchronized public void run() {
- try {
- // logger.debug("Job Fetcher is running...");
- Map<String, Executable> runningJobs = context.getRunningJobs();
- if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
- logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
- return;
- }
-
- int nRunning = 0, nReady = 0, nOthers = 0;
- for (final String id : executableManager.getAllJobIds()) {
- if (runningJobs.containsKey(id)) {
- // logger.debug("Job id:" + id + " is already running");
- nRunning++;
- continue;
- }
- final Output output = executableManager.getOutput(id);
- if ((output.getState() != ExecutableState.READY)) {
- // logger.debug("Job id:" + id + " not runnable");
- nOthers++;
- continue;
- }
- nReady++;
- AbstractExecutable executable = executableManager.getJob(id);
- String jobDesc = executable.toString();
- logger.info(jobDesc + " prepare to schedule");
- try {
- context.addRunningJob(executable);
- jobPool.execute(new JobRunner(executable));
- logger.info(jobDesc + " scheduled");
- } catch (Exception ex) {
- context.removeRunningJob(executable);
- logger.warn(jobDesc + " fail to schedule", ex);
- }
- }
- logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, " + nReady + " ready, " + nOthers + " others");
- } catch (Exception e) {
- logger.warn("Job Fetcher caught a exception " + e);
- }
- }
- }
-
- private class JobRunner implements Runnable {
-
- private final AbstractExecutable executable;
-
- public JobRunner(AbstractExecutable executable) {
- this.executable = executable;
- }
-
- @Override
- public void run() {
- try {
- executable.execute(context);
- // trigger the next step asap
- fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
- } catch (ExecuteException e) {
- logger.error("ExecuteException job:" + executable.getId(), e);
- } catch (Exception e) {
- logger.error("unknown error execute job:" + executable.getId(), e);
- } finally {
- context.removeRunningJob(executable);
- }
- }
- }
-
- public static DefaultScheduler getInstance() {
- return INSTANCE;
- }
-
- @Override
- public void stateChanged(CuratorFramework client, ConnectionState newState) {
- if ((newState == ConnectionState.SUSPENDED) || (newState == ConnectionState.LOST)) {
- try {
- shutdown();
- } catch (SchedulerException e) {
- throw new RuntimeException("failed to shutdown scheduler", e);
- }
- }
- }
-
- @Override
- public synchronized void init(JobEngineConfig jobEngineConfig, final JobLock jobLock) throws SchedulerException {
- if (!initialized) {
- initialized = true;
- } else {
- return;
- }
-
- this.jobEngineConfig = jobEngineConfig;
- jobLock.lock();
-
- executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
- //load all executable, set them to a consistent status
- fetcherPool = Executors.newScheduledThreadPool(1);
- int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
- jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>());
- context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());
-
- for (AbstractExecutable executable : executableManager.getAllExecutables()) {
- if (executable.getStatus() == ExecutableState.READY) {
- executableManager.updateJobOutput(executable.getId(), ExecutableState.ERROR, null, "scheduler initializing work to reset job to ERROR status");
- }
- }
- executableManager.updateAllRunningJobsToError();
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- logger.debug("Closing zk connection");
- try {
- shutdown();
- jobLock.unlock();
- } catch (SchedulerException e) {
- logger.error("error shutdown scheduler", e);
- }
- }
- });
-
- fetcher = new FetcherRunner();
- fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
- hasStarted = true;
- }
-
- @Override
- public void shutdown() throws SchedulerException {
- fetcherPool.shutdown();
- jobPool.shutdown();
- }
-
- @Override
- public boolean stop(AbstractExecutable executable) throws SchedulerException {
- if (hasStarted) {
- return true;
- } else {
- //TODO should try to stop this executable
- return true;
- }
- }
-
- public boolean hasStarted() {
- return this.hasStarted;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
deleted file mode 100644
index 1aa72f8..0000000
--- a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJob.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.invertedindex;
-
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-
-/**
- * Created by shaoshi on 1/15/15.
- */
-public class IIJob extends DefaultChainedExecutable {
-
- public IIJob() {
- super();
- }
-
- private static final String II_INSTANCE_NAME = "iiName";
- private static final String SEGMENT_ID = "segmentId";
-
- void setIIName(String name) {
- setParam(II_INSTANCE_NAME, name);
- }
-
- public String getIIName() {
- return getParam(II_INSTANCE_NAME);
- }
-
- void setSegmentId(String segmentId) {
- setParam(SEGMENT_ID, segmentId);
- }
-
- public String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java b/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
deleted file mode 100644
index 68ad36b..0000000
--- a/job/src/main/java/org/apache/kylin/job/invertedindex/IIJobBuilder.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.invertedindex;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.AbstractJobBuilder;
-import org.apache.kylin.job.common.HadoopShellExecutable;
-import org.apache.kylin.job.common.MapReduceExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.hadoop.dict.CreateInvertedIndexDictionaryJob;
-import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.hadoop.invertedindex.IIBulkLoadJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHFileJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.job.hadoop.invertedindex.IIDistinctColumnsJob;
-import org.apache.kylin.job.hadoop.invertedindex.InvertedIndexJob;
-import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Created by shaoshi on 1/15/15.
- */
-public final class IIJobBuilder extends AbstractJobBuilder {
-
- public IIJobBuilder(JobEngineConfig engineConfig) {
- super(engineConfig);
- }
-
- public IIJob buildJob(IISegment seg) {
- checkPreconditions(seg);
-
- IIJob result = initialJob(seg, "BUILD");
- final String jobId = result.getId();
- final IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(seg.getIIDesc());
- final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId);
- final String factDistinctColumnsPath = getIIDistinctColumnsPath(seg, jobId);
- final String iiRootPath = getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/";
- final String iiPath = iiRootPath + "*";
-
- final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId);
- result.addTask(intermediateHiveTableStep);
-
- result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId, factDistinctColumnsPath));
-
- result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath));
-
- result.addTask(createInvertedIndexStep(seg, intermediateHiveTableName, iiRootPath));
-
- // create htable step
- result.addTask(createCreateHTableStep(seg));
-
- // generate hfiles step
- result.addTask(createConvertToHfileStep(seg, iiPath, jobId));
-
- // bulk load step
- result.addTask(createBulkLoadStep(seg, jobId));
-
- return result;
- }
-
- private IIJob initialJob(IISegment seg, String type) {
- IIJob result = new IIJob();
- SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone(engineConfig.getTimeZone()));
- result.setIIName(seg.getIIInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setName(seg.getIIInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis())));
- result.setSubmitter(this.submitter);
- return result;
- }
-
- private void checkPreconditions(IISegment seg) {
- Preconditions.checkNotNull(seg, "segment cannot be null");
- Preconditions.checkNotNull(engineConfig, "jobEngineConfig cannot be null");
- }
-
- private void appendMapReduceParameters(StringBuilder builder, JobEngineConfig engineConfig) {
- try {
- String jobConf = engineConfig.getHadoopJobConfFilePath(RealizationCapacity.MEDIUM);
- if (jobConf != null && jobConf.length() > 0) {
- builder.append(" -conf ").append(jobConf);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private String getIIDistinctColumnsPath(IISegment seg, String jobUuid) {
- return getJobWorkingDir(jobUuid) + "/" + seg.getIIInstance().getName() + "/ii_distinct_columns";
- }
-
- private String getHFilePath(IISegment seg, String jobId) {
- return getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/hfile/";
- }
-
- private MapReduceExecutable createFactDistinctColumnsStep(IISegment seg, String factTableName, String jobId, String output) {
- MapReduceExecutable result = new MapReduceExecutable();
- result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS);
- result.setMapReduceJobClass(IIDistinctColumnsJob.class);
- StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, engineConfig);
- appendExecCmdParameters(cmd, "tablename", factTableName);
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
- appendExecCmdParameters(cmd, "output", output);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Fact_Distinct_Columns_" + seg.getIIInstance().getName() + "_Step");
-
- result.setMapReduceParams(cmd.toString());
- return result;
- }
-
- private HadoopShellExecutable createBuildDictionaryStep(IISegment seg, String factDistinctColumnsPath) {
- // base cuboid job
- HadoopShellExecutable buildDictionaryStep = new HadoopShellExecutable();
- buildDictionaryStep.setName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY);
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
- appendExecCmdParameters(cmd, "input", factDistinctColumnsPath);
-
- buildDictionaryStep.setJobParams(cmd.toString());
- buildDictionaryStep.setJobClass(CreateInvertedIndexDictionaryJob.class);
- return buildDictionaryStep;
- }
-
- private MapReduceExecutable createInvertedIndexStep(IISegment seg, String intermediateHiveTable, String iiOutputTempPath) {
- // base cuboid job
- MapReduceExecutable buildIIStep = new MapReduceExecutable();
-
- StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, engineConfig);
-
- buildIIStep.setName(ExecutableConstants.STEP_NAME_BUILD_II);
-
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
- appendExecCmdParameters(cmd, "tablename", intermediateHiveTable);
- appendExecCmdParameters(cmd, "output", iiOutputTempPath);
- appendExecCmdParameters(cmd, "jobname", ExecutableConstants.STEP_NAME_BUILD_II);
-
- buildIIStep.setMapReduceParams(cmd.toString());
- buildIIStep.setMapReduceJobClass(InvertedIndexJob.class);
- return buildIIStep;
- }
-
- private HadoopShellExecutable createCreateHTableStep(IISegment seg) {
- HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
- createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
-
- createHtableStep.setJobParams(cmd.toString());
- createHtableStep.setJobClass(IICreateHTableJob.class);
-
- return createHtableStep;
- }
-
- private MapReduceExecutable createConvertToHfileStep(IISegment seg, String inputPath, String jobId) {
- MapReduceExecutable createHFilesStep = new MapReduceExecutable();
- createHFilesStep.setName(ExecutableConstants.STEP_NAME_CONVERT_II_TO_HFILE);
- StringBuilder cmd = new StringBuilder();
-
- appendMapReduceParameters(cmd, engineConfig);
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getHFilePath(seg, jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "jobname", "Kylin_HFile_Generator_" + seg.getIIInstance().getName() + "_Step");
-
- createHFilesStep.setMapReduceParams(cmd.toString());
- createHFilesStep.setMapReduceJobClass(IICreateHFileJob.class);
-
- return createHFilesStep;
- }
-
- private HadoopShellExecutable createBulkLoadStep(IISegment seg, String jobId) {
- HadoopShellExecutable bulkLoadStep = new HadoopShellExecutable();
- bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE);
-
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "input", getHFilePath(seg, jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName());
-
- bulkLoadStep.setJobParams(cmd.toString());
- bulkLoadStep.setJobClass(IIBulkLoadJob.class);
-
- return bulkLoadStep;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
deleted file mode 100644
index 3c79f8c..0000000
--- a/job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.manager;
-
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.job.dao.ExecutableDao;
-import org.apache.kylin.job.dao.ExecutableOutputPO;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.apache.kylin.job.exception.IllegalStateTranferException;
-import org.apache.kylin.job.exception.PersistentException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ChainedExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.DefaultOutput;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- */
-public class ExecutableManager {
-
- private static final Logger logger = LoggerFactory.getLogger(ExecutableManager.class);
- private static final ConcurrentHashMap<KylinConfig, ExecutableManager> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableManager>();
- @SuppressWarnings("unused")
- private final KylinConfig config;
-
- private ExecutableDao executableDao;
-
- public static ExecutableManager getInstance(KylinConfig config) {
- ExecutableManager r = CACHE.get(config);
- if (r == null) {
- r = new ExecutableManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
-
- }
- return r;
- }
-
- private ExecutableManager(KylinConfig config) {
- logger.info("Using metadata url: " + config);
- this.config = config;
- this.executableDao = ExecutableDao.getInstance(config);
- }
-
- public void addJob(AbstractExecutable executable) {
- try {
- executableDao.addJob(parse(executable));
- addJobOutput(executable);
- } catch (PersistentException e) {
- logger.error("fail to submit job:" + executable.getId(), e);
- throw new RuntimeException(e);
- }
- }
-
- private void addJobOutput(AbstractExecutable executable) throws PersistentException {
- ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
- executableOutputPO.setUuid(executable.getId());
- executableDao.addJobOutput(executableOutputPO);
- if (executable instanceof DefaultChainedExecutable) {
- for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) {
- addJobOutput(subTask);
- }
- }
- }
-
- //for ut
- public void deleteJob(String jobId) {
- try {
- executableDao.deleteJob(jobId);
- } catch (PersistentException e) {
- logger.error("fail to delete job:" + jobId, e);
- throw new RuntimeException(e);
- }
- }
-
- public AbstractExecutable getJob(String uuid) {
- try {
- return parseTo(executableDao.getJob(uuid));
- } catch (PersistentException e) {
- logger.error("fail to get job:" + uuid, e);
- throw new RuntimeException(e);
- }
- }
-
- public Output getOutput(String uuid) {
- try {
- final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
- Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
- return parseOutput(jobOutput);
- } catch (PersistentException e) {
- logger.error("fail to get job output:" + uuid, e);
- throw new RuntimeException(e);
- }
- }
-
- private DefaultOutput parseOutput(ExecutableOutputPO jobOutput) {
- final DefaultOutput result = new DefaultOutput();
- result.setExtra(jobOutput.getInfo());
- result.setState(ExecutableState.valueOf(jobOutput.getStatus()));
- result.setVerboseMsg(jobOutput.getContent());
- result.setLastModified(jobOutput.getLastModified());
- return result;
- }
-
- public Map<String, Output> getAllOutputs() {
- try {
- final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
- HashMap<String, Output> result = Maps.newHashMap();
- for (ExecutableOutputPO jobOutput : jobOutputs) {
- result.put(jobOutput.getId(), parseOutput(jobOutput));
- }
- return result;
- } catch (PersistentException e) {
- logger.error("fail to get all job output:", e);
- throw new RuntimeException(e);
- }
- }
-
- public List<AbstractExecutable> getAllExecutables() {
- try {
- List<AbstractExecutable> ret = Lists.newArrayList();
- for (ExecutablePO po : executableDao.getJobs()) {
- try {
- AbstractExecutable ae = parseTo(po);
- ret.add(ae);
- } catch (IllegalArgumentException e) {
- logger.error("error parsing one executabePO: ", e);
- }
- }
- return ret;
- } catch (PersistentException e) {
- logger.error("error get All Jobs", e);
- throw new RuntimeException(e);
- }
- }
-
- public List<String> getAllJobIds() {
- try {
- return executableDao.getJobIds();
- } catch (PersistentException e) {
- logger.error("error get All Job Ids", e);
- throw new RuntimeException(e);
- }
- }
-
- public void updateAllRunningJobsToError() {
- try {
- final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
- for (ExecutableOutputPO executableOutputPO : jobOutputs) {
- if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
- executableOutputPO.setStatus(ExecutableState.ERROR.toString());
- executableDao.updateJobOutput(executableOutputPO);
- }
- }
- } catch (PersistentException e) {
- logger.error("error reset job status from RUNNING to ERROR", e);
- throw new RuntimeException(e);
- }
- }
-
- public void resumeJob(String jobId) {
- AbstractExecutable job = getJob(jobId);
- if (job == null) {
- return;
- }
- updateJobOutput(jobId, ExecutableState.READY, null, null);
- if (job instanceof DefaultChainedExecutable) {
- List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks();
- for (AbstractExecutable task : tasks) {
- if (task.getStatus() == ExecutableState.ERROR) {
- updateJobOutput(task.getId(), ExecutableState.READY, null, null);
- break;
- }
- }
- }
- }
-
- public void discardJob(String jobId) {
- AbstractExecutable job = getJob(jobId);
- if (job instanceof DefaultChainedExecutable) {
- List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks();
- for (AbstractExecutable task : tasks) {
- if (!task.getStatus().isFinalState()) {
- updateJobOutput(task.getId(), ExecutableState.DISCARDED, null, null);
- }
- }
- }
- updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
- }
-
- public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) {
- try {
- final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
- Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId);
- ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus());
- if (newStatus != null && oldStatus != newStatus) {
- if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) {
- throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus);
- }
- jobOutput.setStatus(newStatus.toString());
- }
- if (info != null) {
- jobOutput.setInfo(info);
- }
- if (output != null) {
- jobOutput.setContent(output);
- }
- executableDao.updateJobOutput(jobOutput);
- logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus);
- } catch (PersistentException e) {
- logger.error("error change job:" + jobId + " to " + newStatus.toString());
- throw new RuntimeException(e);
- }
- }
-
- //for migration only
- //TODO delete when migration finished
- public void resetJobOutput(String jobId, ExecutableState state, String output) {
- try {
- final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
- jobOutput.setStatus(state.toString());
- if (output != null) {
- jobOutput.setContent(output);
- }
- executableDao.updateJobOutput(jobOutput);
- } catch (PersistentException e) {
- throw new RuntimeException(e);
- }
- }
-
- public void addJobInfo(String id, Map<String, String> info) {
- if (info == null) {
- return;
- }
- try {
- ExecutableOutputPO output = executableDao.getJobOutput(id);
- Preconditions.checkArgument(output != null, "there is no related output for job id:" + id);
- output.getInfo().putAll(info);
- executableDao.updateJobOutput(output);
- } catch (PersistentException e) {
- logger.error("error update job info, id:" + id + " info:" + info.toString());
- throw new RuntimeException(e);
- }
- }
-
- public void addJobInfo(String id, String key, String value) {
- Map<String, String> info = Maps.newHashMap();
- info.put(key, value);
- addJobInfo(id, info);
- }
-
- private static ExecutablePO parse(AbstractExecutable executable) {
- ExecutablePO result = new ExecutablePO();
- result.setName(executable.getName());
- result.setUuid(executable.getId());
- result.setType(executable.getClass().getName());
- result.setParams(executable.getParams());
- if (executable instanceof ChainedExecutable) {
- List<ExecutablePO> tasks = Lists.newArrayList();
- for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
- tasks.add(parse(task));
- }
- result.setTasks(tasks);
- }
- return result;
- }
-
- private static AbstractExecutable parseTo(ExecutablePO executablePO) {
- if (executablePO == null) {
- return null;
- }
- String type = executablePO.getType();
- try {
- Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class);
- Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
- AbstractExecutable result = constructor.newInstance();
- result.setId(executablePO.getUuid());
- result.setName(executablePO.getName());
- result.setParams(executablePO.getParams());
- List<ExecutablePO> tasks = executablePO.getTasks();
- if (tasks != null && !tasks.isEmpty()) {
- Preconditions.checkArgument(result instanceof ChainedExecutable);
- for (ExecutablePO subTask : tasks) {
- ((ChainedExecutable) result).addTask(parseTo(subTask));
- }
- }
- return result;
- } catch (ReflectiveOperationException e) {
- throw new IllegalArgumentException("cannot parse this job:" + executablePO.getId(), e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
deleted file mode 100644
index b6e5af5..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/CleanHtableCLI.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by honma on 11/11/14.
- */
-public class CleanHtableCLI extends AbstractHadoopJob {
-
- protected static final Logger log = LoggerFactory.getLogger(CleanHtableCLI.class);
-
- String tableName;
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
- try {
-
- clean();
-
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- private void clean() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
-
- for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
- String name = descriptor.getNameAsString().toLowerCase();
- if (name.startsWith("kylin") || name.startsWith("_kylin")) {
- String x = descriptor.getValue("KYLIN_HOST");
- System.out.println("table name " + descriptor.getNameAsString() + " host: " + x);
- System.out.println(descriptor);
- System.out.println();
- }
- }
- hbaseAdmin.close();
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new CleanHtableCLI(), args);
- System.exit(exitCode);
- }
-}
[25/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/cube/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube/kylin_sales_cube.json b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
deleted file mode 100644
index 5c36aa1..0000000
--- a/examples/sample_cube/metadata/cube/kylin_sales_cube.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "kylin_sales_cube",
- "owner" : null,
- "version" : null,
- "descriptor" : "kylin_sales_cube_desc",
- "status" : "DISABLED",
- "segments" : [ ],
- "auto_merge_time_ranges" : [604800000, 2419200000],
- "retention_range": 63072000000,
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
deleted file mode 100644
index 4d402fe..0000000
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ /dev/null
@@ -1,166 +0,0 @@
-{
- "uuid" : "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "kylin_sales_cube_desc",
- "description" : null,
- "dimensions" : [ {
- "id" : 0,
- "name" : "CAL_DT",
- "table" : "DEFAULT.KYLIN_CAL_DT",
- "column" : null,
- "derived" : [ "WEEK_BEG_DT" ],
- "hierarchy" : false
- }, {
- "id" : 1,
- "name" : "CATEGORY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : null,
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ],
- "hierarchy" : false
- }, {
- "id" : 2,
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ],
- "derived" : null,
- "hierarchy" : true
- }, {
- "id" : 3,
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.KYLIN_SALES",
- "column" : [ "LSTG_FORMAT_NAME" ],
- "derived" : null,
- "hierarchy" : false
- } ],
- "measures" : [ {
- "id" : 1,
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "id" : 2,
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "id" : 3,
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "id" : 4,
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1"
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }, {
- "id" : 5,
- "name" : "SELLER_CNT_HLL",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID"
- },
- "returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
- }, {
- "id" : 6,
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME"
- },
- "returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "part_dt",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- }, {
- "column" : "leaf_categ_id",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- }, {
- "column" : "meta_categ_name",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- }, {
- "column" : "categ_lvl2_name",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- }, {
- "column" : "categ_lvl3_name",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- }, {
- "column" : "lstg_format_name",
- "length" : 12,
- "dictionary" : null,
- "mandatory" : false
- }, {
- "column" : "lstg_site_id",
- "length" : 0,
- "dictionary" : "true",
- "mandatory" : false
- } ],
- "aggregation_groups" : [ [ "part_dt", "lstg_site_id", "leaf_categ_id", "meta_categ_name", "categ_lvl3_name", "categ_lvl2_name", "lstg_format_name" ] ]
- },
- "signature" : "OWc6h/qpWCBgxEnzB5wr0w==",
- "last_modified" : 1426255280419,
- "model_name" : "kylin_sales_model",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt" ]
- } ]
- }, {
- "name" : "f2",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "seller_cnt_hll", "seller_format_cnt" ]
- } ]
- } ]
- },
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json b/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
deleted file mode 100644
index 157a036..0000000
--- a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "name" : "kylin_sales_model",
- "lookups" : [ {
- "table" : "DEFAULT.KYLIN_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "PART_DT" ]
- }
- }, {
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }],
- "capacity" : "SMALL",
- "last_modified" : 1422435345362,
- "fact_table" : "DEFAULT.KYLIN_SALES",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.KYLIN_SALES.PART_DT",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/project/learn_kylin.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/project/learn_kylin.json b/examples/sample_cube/metadata/project/learn_kylin.json
deleted file mode 100644
index cf082f5..0000000
--- a/examples/sample_cube/metadata/project/learn_kylin.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
- "name": "learn_kylin",
- "realizations": [
- {
- "name": "kylin_sales_cube",
- "type": "CUBE",
- "realization": "kylin_sales_cube"
- }],
- "tables": ["DEFAULT.KYLIN_SALES", "DEFAULT.KYLIN_CAL_DT", "DEFAULT.KYLIN_CATEGORY_GROUPINGS"]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
deleted file mode 100644
index 5bdd778..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
+++ /dev/null
@@ -1,407 +0,0 @@
-{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "KYLIN_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "default",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
deleted file mode 100644
index d1461b7..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ /dev/null
@@ -1,151 +0,0 @@
-{
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
- "name" : "KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
deleted file mode 100644
index be7cde5..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
- "name" : "KYLIN_SALES",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "PART_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal(19,4)"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "bigint"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
deleted file mode 100644
index 95c9231..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_1_new_segment.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "111ca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 1404098141020,
- "name" : "test_kylin_cube_with_slr_1_new_segment",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_desc",
- "cost" : 50,
- "segments" : [ {
- "name" : "20130331080000_20131212080000",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
- "date_range_start" : 1364688000000,
- "date_range_end" : 1386806400000,
- "status" : "NEW",
- "size_kb" : 0,
- "source_records" : 0,
- "source_records_size" : 0,
- "last_build_time" : 0,
- "last_build_job_id" : null,
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- } ],
- "status" : "DISABLED",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
deleted file mode 100644
index 70d89ea..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "test_kylin_cube_with_slr_empty",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
deleted file mode 100644
index e708a86..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_empty.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8dddd",
- "last_modified" : 0,
- "name" : "test_kylin_cube_with_slr_left_join_empty",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_left_join_desc",
- "segments" : [ ],
- "status" : "DISABLED",
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
deleted file mode 100644
index 5f15336..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_left_join_ready.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "kkkka32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 1404097095621,
- "name" : "test_kylin_cube_with_slr_left_join_ready",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_left_join_desc",
- "cost" : 50,
- "segments" : [ {
- "name" : null,
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_LEFT_JOIN_READY-BCF2F125-9B0B-40DD-9509-95EC59B31333",
- "date_range_start" : null,
- "date_range_end" : null,
- "status" : "READY",
- "size_kb" : 7690,
- "source_records" : 10000,
- "source_records_size" : 610288,
- "last_build_time" : 1404097095455,
- "last_build_job_id" : "bcf2f125-9b0b-40dd-9509-95ec59b31333",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot"
- }
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
deleted file mode 100644
index 64aae3f..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 1404098141020,
- "name" : "test_kylin_cube_with_slr_ready",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_desc",
- "cost" : 50,
- "segments" : [ {
- "name" : "20130331080000_20131212080000",
- "uuid" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
- "date_range_start" : 1364688000000,
- "date_range_end" : 1386806400000,
- "status" : "READY",
- "size_kb" : 7801,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098140902,
- "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
- "binary_signature" : null,
-
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
-
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
deleted file mode 100644
index 0700449..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_2_segments.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c53c",
- "last_modified" : 1404098141020,
- "name" : "test_kylin_cube_with_slr_ready_2_segments",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_with_slr_desc",
- "cost" : 50,
- "segments" : [ {
- "name" : "19691231160000_20131112000000",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
- "date_range_start" : 1384240200000,
- "date_range_end" : 1384243200000,
- "status" : "READY",
- "size_kb" : 7801,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098140902,
- "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- }, {
- "name" : "20131112000000_20131212000000",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
- "date_range_start" : 1384243200000,
- "date_range_end" : 1386835200000,
- "status" : "READY",
- "size_kb" : 7801,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098140902,
- "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
deleted file mode 100644
index 6bd566a..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "daa53e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 0,
- "name" : "test_kylin_cube_without_slr_empty",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_without_slr_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
deleted file mode 100644
index b4b5fd7..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_empty.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "uuid" : "dddd3e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 0,
- "name" : "test_kylin_cube_without_slr_left_join_empty",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_without_slr_left_join_desc",
- "segments" : [ ],
- "status" : "DISABLED",
- "create_time" : null
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
deleted file mode 100644
index d142700..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "mmmm3e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 1404097288087,
- "name" : "test_kylin_cube_without_slr_left_join_ready",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_without_slr_left_join_desc",
- "cost" : 10,
- "segments" : [ {
- "name" : null,
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITHOUT_SLR_LEFT_JOIN_READY-1EF30C45-17A4-4350-A032-A33B3B25E5B8",
- "date_range_start" : null,
- "date_range_end" : null,
- "status" : "READY",
- "size_kb" : 6060,
- "source_records" : 10000,
- "source_records_size" : 610288,
- "last_build_time" : 1404097287905,
- "last_build_job_id" : "1ef30c45-17a4-4350-a032-a33b3b25e5b8",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/e5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot"
- }
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
deleted file mode 100644
index 0456e66..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_left_join_ready_2_segments.json
+++ /dev/null
@@ -1,74 +0,0 @@
-{
- "uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c5ff",
- "last_modified" : 1404098141020,
- "name" : "test_kylin_cube_without_slr_left_join_ready_2_segments",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_without_slr_left_join_desc",
- "cost" : 50,
- "segments" : [ {
- "name" : "19691231160000_20131112000000",
- "uuid" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
- "date_range_start" : 0,
- "date_range_end" : 1384243200000,
- "status" : "READY",
- "size_kb" : 7801,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098140902,
- "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- }, {
- "name" : "20131112000000_20131212000000",
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
- "date_range_start" : 1384243200000,
- "date_range_end" : 1386835200000,
- "status" : "READY",
- "size_kb" : 7801,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098140902,
- "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
- "TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
deleted file mode 100644
index 1995585..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_ready.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "daa53e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 1404098303976,
- "name" : "test_kylin_cube_without_slr_ready",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_without_slr_desc",
- "cost" : 10,
- "segments" : [ {
- "name" : null,
- "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITHOUT_SLR_READY-619D9B4A-DA56-48CA-9C58-29DC8323B200",
- "date_range_start" : null,
- "date_range_end" : null,
- "status" : "READY",
- "size_kb" : 5600,
- "source_records" : 10000,
- "source_records_size" : 608012,
- "last_build_time" : 1404098303829,
- "last_build_job_id" : "619d9b4a-da56-48ca-9c58-29dc8323b200",
- "binary_signature" : null,
- "dictionaries" : {
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
- "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict",
- "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict"
- },
- "snapshots" : {
- "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
- "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/9a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot",
- "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
- "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
- }
- } ],
- "status" : "READY",
- "create_time" : null,
- "notify_list" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
deleted file mode 100644
index 4a2a4dd..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ /dev/null
@@ -1,241 +0,0 @@
-{
- "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_with_slr_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 2,
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": null,
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": null,
- "derived": [
- "SELLER_TYPE_DESC"
- ],
- "hierarchy": false
- },
- {
- "id": 6,
- "name": "SELLER_ID",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "SELLER_ID"
- ],
- "derived": null,
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "seller_id",
- "length": 18,
- "dictionary": null,
- "mandatory": true
- },
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "slr_segment_cd",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "cal_dt"
- ]
- ]
- },
- "signature": "lsLAl2jL62ZApmOLZqWU3g==",
- "last_modified": 1422435345330,
- "model_name": "test_kylin_with_slr_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
deleted file mode 100644
index 8e6b2ff..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ /dev/null
@@ -1,241 +0,0 @@
-{
- "uuid": "bbbba905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_with_slr_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 2,
- "name": "CATEGORY_DERIVED",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": null,
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": null,
- "derived": [
- "SELLER_TYPE_DESC"
- ],
- "hierarchy": false
- },
- {
- "id": 6,
- "name": "SELLER_ID",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "SELLER_ID"
- ],
- "derived": null,
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "seller_id",
- "length": 18,
- "dictionary": null,
- "mandatory": true
- },
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "slr_segment_cd",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name",
- "cal_dt"
- ]
- ]
- },
- "signature": "ljba0vaTnt00lU4rdhG9Xw==",
- "last_modified": 1422435345352,
- "model_name": "test_kylin_with_slr_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
deleted file mode 100644
index f31e358..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ /dev/null
@@ -1,287 +0,0 @@
-{
- "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_without_slr_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 2,
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": null,
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": null,
- "derived": [
- "SELLER_TYPE_DESC"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "SELLER_CNT_HLL",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID"
- },
- "returntype": "hllc(10)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 7,
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "hllc(10)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 8,
- "name": "LEAF_CATEG_ID_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LEAF_CATEG_ID"
- },
- "returntype": "bitmap"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": true
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "slr_segment_cd",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "lstg_format_name",
- "lstg_site_id",
- "slr_segment_cd"
- ],
- [
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl3_name",
- "categ_lvl2_name",
- "lstg_format_name"
- ]
- ]
- },
- "signature": "7qJiv2MEGoGf1AAgksQmZw==",
- "last_modified": 1422435345362,
- "model_name": "test_kylin_without_slr_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "seller_cnt_hll",
- "seller_format_cnt",
- "leaf_categ_id_bitmap"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
deleted file mode 100644
index ef27b03..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ /dev/null
@@ -1,287 +0,0 @@
-{
- "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name": "test_kylin_cube_without_slr_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- },
- {
- "id": 1,
- "name": "CATEGORY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 2,
- "name": "CATEGORY_HIERARCHY",
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column": [
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME"
- ],
- "derived": null,
- "hierarchy": true
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "table": "DEFAULT.TEST_KYLIN_FACT",
- "column": [
- "LSTG_FORMAT_NAME"
- ],
- "derived": null,
- "hierarchy": false
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "table": "EDW.TEST_SITES",
- "column": null,
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ],
- "hierarchy": false
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "column": null,
- "derived": [
- "SELLER_TYPE_DESC"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "SELLER_CNT_HLL",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID"
- },
- "returntype": "hllc(10)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 7,
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "hllc(10)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 8,
- "name": "LEAF_CATEG_ID_BITMAP",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LEAF_CATEG_ID"
- },
- "returntype": "bitmap"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": true
- },
- {
- "column": "leaf_categ_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "meta_categ_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl2_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "categ_lvl3_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_format_name",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "lstg_site_id",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- },
- {
- "column": "slr_segment_cd",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "lstg_format_name",
- "lstg_site_id",
- "slr_segment_cd"
- ],
- [
- "leaf_categ_id",
- "meta_categ_name",
- "categ_lvl3_name",
- "categ_lvl2_name",
- "lstg_format_name"
- ]
- ]
- },
- "signature": "TPdd/nnYLZnsI8TtZzoCng==",
- "last_modified": 1422435345373,
- "model_name": "test_kylin_without_slr_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "seller_cnt_hll",
- "seller_format_cnt",
- "leaf_categ_id_bitmap"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/.gitignore
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/.gitignore b/examples/test_case_data/localmeta/data/.gitignore
deleted file mode 100644
index 87d7b53..0000000
--- a/examples/test_case_data/localmeta/data/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.csv.back
[09/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
deleted file mode 100644
index 56c9659..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- */
-public class FactDistinctColumnsJob extends AbstractHadoopJob {
- protected static final Logger log = LoggerFactory.getLogger(FactDistinctColumnsJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_TABLE_NAME);
- parseOptions(options, args);
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- Configuration jobConf = job.getConfiguration();
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME);
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String intermediateTable = getOptionValue(OPTION_TABLE_NAME);
-
- // ----------------------------------------------------------------------------
- // add metadata to distributed cache
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
-
- jobConf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
- System.out.println("Starting: " + job.getJobName());
-
- setJobClasspath(job);
-
- setupMapper(intermediateTable);
- setupReducer(output);
-
- // CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
- attachKylinPropsAndMetadata(cubeInstance, job.getConfiguration());
-
- return waitForCompletion(job);
-
- } catch (Exception e) {
- logger.error("error in FactDistinctColumnsJob", e);
- printUsage(options);
- throw e;
- } finally {
- if (job != null) {
- cleanupTempConfFile(job.getConfiguration());
- }
- }
-
- }
-
- private void setupMapper(String intermediateTable) throws IOException {
- // FileInputFormat.setInputPaths(job, input);
-
- String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
- HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);
-
- job.setInputFormatClass(HCatInputFormat.class);
- job.setMapperClass(FactDistinctColumnsMapper.class);
- job.setCombinerClass(FactDistinctColumnsCombiner.class);
- job.setMapOutputKeyClass(ShortWritable.class);
- job.setMapOutputValueClass(Text.class);
- }
-
- private void setupReducer(Path output) throws IOException {
- job.setReducerClass(FactDistinctColumnsReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Text.class);
-
- FileOutputFormat.setOutputPath(job, output);
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
-
- job.setNumReduceTasks(1);
-
- deletePath(job.getConfiguration(), output);
- }
-
- public static void main(String[] args) throws Exception {
- FactDistinctColumnsJob job = new FactDistinctColumnsJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
deleted file mode 100644
index 72802aa..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- */
-public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, ShortWritable, Text> {
-
- private String cubeName;
- private CubeInstance cube;
- private CubeDesc cubeDesc;
- private int[] factDictCols;
-
- private CubeJoinedFlatTableDesc intermediateTableDesc;
-
- private ShortWritable outputKey = new ShortWritable();
- private Text outputValue = new Text();
- private int errorRecordCounter;
-
- private HCatSchema schema = null;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
- cube = CubeManager.getInstance(config).getCube(cubeName);
- cubeDesc = cube.getDescriptor();
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- List<TblColRef> columns = baseCuboid.getColumns();
-
- ArrayList<Integer> factDictCols = new ArrayList<Integer>();
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- for (int i = 0; i < columns.size(); i++) {
- TblColRef col = columns.get(i);
- if (rowkey.isUseDictionary(col) == false)
- continue;
-
- String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
- if (cubeDesc.getModel().isFactTable(scanTable)) {
- factDictCols.add(i);
- }
- }
- this.factDictCols = new int[factDictCols.size()];
- for (int i = 0; i < factDictCols.size(); i++)
- this.factDictCols[i] = factDictCols.get(i);
-
- schema = HCatInputFormat.getTableSchema(context.getConfiguration());
- }
-
- @Override
- public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {
-
- try {
-
- int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
- HCatFieldSchema fieldSchema = null;
- for (int i : factDictCols) {
- outputKey.set((short) i);
- fieldSchema = schema.get(flatTableIndexes[i]);
- Object fieldValue = record.get(fieldSchema.getName(), schema);
- if (fieldValue == null)
- continue;
- byte[] bytes = Bytes.toBytes(fieldValue.toString());
- outputValue.set(bytes, 0, bytes.length);
- context.write(outputKey, outputValue);
- }
- } catch (Exception ex) {
- handleErrorRecord(record, ex);
- }
-
- }
-
- private void handleErrorRecord(HCatRecord record, Exception ex) throws IOException {
-
- System.err.println("Insane record: " + record.getAll());
- ex.printStackTrace(System.err);
-
- errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
- if (ex instanceof IOException)
- throw (IOException) ex;
- else if (ex instanceof RuntimeException)
- throw (RuntimeException) ex;
- else
- throw new RuntimeException("", ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
deleted file mode 100644
index 89f90ba..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-/**
- * @author yangli9
- */
-public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text, NullWritable, Text> {
-
- private List<TblColRef> columnList = new ArrayList<TblColRef>();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
- CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
- CubeDesc cubeDesc = cube.getDescriptor();
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- columnList = baseCuboid.getColumns();
- }
-
- @Override
- public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
- TblColRef col = columnList.get(key.get());
-
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- }
-
- Configuration conf = context.getConfiguration();
- FileSystem fs = FileSystem.get(conf);
- String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
- FSDataOutputStream out = fs.create(new Path(outputPath, col.getName()));
-
- try {
- for (ByteArray value : set) {
- out.write(value.data);
- out.write('\n');
- }
- } finally {
- out.close();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
deleted file mode 100644
index befa16f..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import org.apache.commons.cli.Options;
-//import org.apache.hadoop.fs.FileSystem;
-//import org.apache.hadoop.fs.Path;
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mapreduce.Job;
-//import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-//import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-//import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-//import org.apache.hadoop.util.ToolRunner;
-//
-//import org.apache.kylin.cube.CubeInstance;
-//import org.apache.kylin.cube.CubeManager;
-//import org.apache.kylin.cube.cuboid.Cuboid;
-//import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
-//import org.apache.kylin.cube.kv.RowKeyEncoder;
-//import org.apache.kylin.index.AbstractHadoopJob;
-//import org.apache.kylin.metadata.model.cube.CubeDesc;
-//
-///**
-// * @author xjiang
-// *
-// */
-//
-//public class KeyDistributionJob extends AbstractHadoopJob {
-//
-// public static final String JOB_TITLE = "Kylin Row Key Distribution Job";
-// public static final String KEY_HEADER_LENGTH = "key_header_length";
-// public static final String KEY_COLUMN_PERCENTAGE = "key_column_percentage";
-// public static final String KEY_SPLIT_NUMBER = "key_split_number";
-//
-// /* (non-Javadoc)
-// * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
-// */
-// @Override
-// public int run(String[] args) throws Exception {
-// Options options = new Options();
-//
-// try {
-// options.addOption(OPTION_INPUT_PATH);
-// options.addOption(OPTION_OUTPUT_PATH);
-// options.addOption(OPTION_METADATA_URL);
-// options.addOption(OPTION_CUBE_NAME);
-// options.addOption(OPTION_KEY_COLUMN_PERCENTAGE);
-// options.addOption(OPTION_KEY_SPLIT_NUMBER);
-// parseOptions(options, args);
-//
-// // start job
-// String jobName = JOB_TITLE + getOptionsAsString();
-// System.out.println("Starting: " + jobName);
-// Job job = Job.getInstanceFromEnv(getConf(), jobName);
-//
-// // set job configuration - basic
-// setJobClasspath(job);
-// addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
-//
-// Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-// FileOutputFormat.setOutputPath(job, output);
-// //job.getConfiguration().set("dfs.block.size", "67108864");
-//
-// // set job configuration - key prefix size & key split number
-// String keyColumnPercentage = getOptionValue(OPTION_KEY_COLUMN_PERCENTAGE);
-// job.getConfiguration().set(KEY_COLUMN_PERCENTAGE, keyColumnPercentage);
-// String metadataUrl = validateMetadataUrl(getOptionValue(OPTION_METADATA_URL));
-// String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
-// int keyHeaderLen = getKeyHeaderLength(metadataUrl, cubeName);
-// job.getConfiguration().set(KEY_HEADER_LENGTH, String.valueOf(keyHeaderLen));
-// job.getConfiguration().set(KEY_SPLIT_NUMBER, getOptionValue(OPTION_KEY_SPLIT_NUMBER));
-//
-// // Mapper
-// job.setInputFormatClass(SequenceFileInputFormat.class);
-// job.setMapperClass(KeyDistributionMapper.class);
-// job.setMapOutputKeyClass(Text.class);
-// job.setMapOutputValueClass(LongWritable.class);
-//
-// // Combiner, not needed any more as mapper now does the groping
-// //job.setCombinerClass(KeyDistributionCombiner.class);
-//
-// // Reducer - only one
-// job.setReducerClass(KeyDistributionReducer.class);
-// // use sequence file as output
-// job.setOutputFormatClass(SequenceFileOutputFormat.class);
-// // key is text
-// job.setOutputKeyClass(Text.class);
-// // value is long
-// job.setOutputValueClass(LongWritable.class);
-// job.setNumReduceTasks(1);
-//
-// FileSystem fs = FileSystem.get(job.getConfiguration());
-// if (fs.exists(output))
-// fs.delete(output, true);
-//
-// return waitForCompletion(job);
-// } catch (Exception e) {
-// printUsage(options);
-// e.printStackTrace(System.err);
-// return 2;
-// }
-// }
-//
-// private int getKeyHeaderLength(String metadataUrl, String cubeName) {
-// CubeManager cubeMgr = CubeManager.getInstanceFromEnv(metadataUrl);
-// CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
-// CubeDesc cubeDesc = cubeInstance.getDescriptor();
-// long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-// Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-// RowKeyEncoder rowKeyEncoder =
-// (RowKeyEncoder) AbstractRowKeyEncoder.createInstance(cubeInstance.getTheOnlySegment(),
-// baseCuboid);
-//
-// return rowKeyEncoder.getHeaderLength();
-//
-// }
-//
-// public static void main(String[] args) throws Exception {
-// int exitCode = ToolRunner.run(new KeyDistributionJob(), args);
-// System.exit(exitCode);
-// }
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapper.java
deleted file mode 100644
index f145bde..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapper.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import java.io.IOException;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mapreduce.Mapper;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionMapper extends KylinMapper<Text, Text, Text, LongWritable> {
-//
-// private int headerLength;
-//
-// private Text currentKey;
-// private long outputLong;
-// private Text outputKey;
-// private LongWritable outputValue;
-// private int columnPercentage;
-// private int allRowCount;
-//
-// @Override
-// protected void setup(Context context) throws IOException {
-//super.publishConfiguration(context.getConfiguration());
-
-// String percentStr = context.getConfiguration().get(KeyDistributionJob.KEY_COLUMN_PERCENTAGE);
-// this.columnPercentage = Integer.valueOf(percentStr).intValue();
-// if (this.columnPercentage <= 0 || this.columnPercentage >= 100) {
-// this.columnPercentage = 20;
-// }
-// String headerLenStr = context.getConfiguration().get(KeyDistributionJob.KEY_HEADER_LENGTH);
-// this.headerLength = Integer.valueOf(headerLenStr).intValue();
-//
-// currentKey = new Text();
-// outputLong = 0;
-// outputKey = new Text();
-// outputValue = new LongWritable(1);
-// allRowCount = 0;
-// }
-//
-// @Override
-// protected void cleanup(Context context) throws IOException, InterruptedException {
-// emit(context); // emit the last holding record
-//
-// byte[] zerokey = new byte[] { 0 };
-// outputKey.set(zerokey);
-// outputValue.set(allRowCount);
-// context.write(outputKey, outputValue);
-// }
-//
-// @Override
-// public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
-// byte[] bytes = key.getBytes();
-// int columnLength = bytes.length - this.headerLength;
-// int columnPrefixLen = columnLength * this.columnPercentage / 100;
-// if (columnPrefixLen == 0 && columnLength > 0) {
-// columnPrefixLen = 1;
-// }
-// if (columnPrefixLen > 0) {
-// currentKey.set(bytes, 0, this.headerLength + columnPrefixLen);
-// } else {
-// currentKey.set(bytes);
-// }
-//
-// allRowCount++;
-//
-// if (outputKey.getLength() == 0) { // first record
-// outputKey.set(currentKey);
-// outputLong = 1;
-// } else if (outputKey.equals(currentKey)) { // same key, note input is sorted
-// outputLong++;
-// } else { // the next key
-// emit(context);
-// outputKey.set(currentKey);
-// outputLong = 1;
-// }
-// }
-//
-// private void emit(Context context) throws IOException, InterruptedException {
-// if (outputLong == 0)
-// return;
-//
-// outputValue.set(outputLong);
-// context.write(outputKey, outputValue);
-// }
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionReducer.java
deleted file mode 100644
index dd02910..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionReducer.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import java.io.IOException;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mapreduce.Reducer;
-//import org.apache.hadoop.util.StringUtils;
-//import org.slf4j.Logger;
-//import org.slf4j.LoggerFactory;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionReducer extends KylinReducer<Text, LongWritable, Text, LongWritable> {
-//
-// private static final Logger logger = LoggerFactory.getLogger(KeyDistributionReducer.class);
-//
-// private LongWritable outputValue;
-// private boolean isTotalCount;
-// private long totalCount;
-// private int splitNumber;
-// private long splitQuota;
-// private long splitRemain;
-//
-// @Override
-// protected void setup(Context context) throws IOException, InterruptedException {
-// super.publishConfiguration(context.getConfiguration());
-
-// String splitStr = context.getConfiguration().get(KeyDistributionJob.KEY_SPLIT_NUMBER);
-// splitNumber = Integer.valueOf(splitStr).intValue();
-// outputValue = new LongWritable();
-// isTotalCount = true;
-// totalCount = 0;
-// splitQuota = 0;
-// splitRemain = 0;
-// }
-//
-// @Override
-// protected void cleanup(Context context) throws IOException, InterruptedException {
-// logger.info("---------------");
-// long splitCount = splitQuota - splitRemain;
-// logger.info("Total Count = " + totalCount + ", Left Count = " + splitCount);
-// }
-//
-// @Override
-// public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException,
-// InterruptedException {
-//
-// // calculate split quota
-// if (isTotalCount) {
-// for (LongWritable count : values) {
-// totalCount += count.get();
-// }
-// splitQuota = totalCount / splitNumber;
-// splitRemain = splitQuota;
-// isTotalCount = false;
-// return;
-// }
-//
-// // output key when split quota is used up
-// for (LongWritable count : values) {
-// splitRemain -= count.get();
-// }
-// if (splitRemain <= 0) {
-// long splitCount = splitQuota - splitRemain;
-// String hexKey = StringUtils.byteToHexString(key.getBytes());
-// logger.info(hexKey + "\t\t" + splitCount);
-//
-// outputValue.set(splitCount);
-// context.write(key, outputValue);
-// splitRemain = splitQuota;
-// }
-//
-// }
-// }
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
deleted file mode 100644
index 73faa6c..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author ysong1
- */
-public class MergeCuboidJob extends CuboidJob {
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_SEGMENT_NAME);
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- parseOptions(options, args);
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
- String segmentName = getOptionValue(OPTION_SEGMENT_NAME).toUpperCase();
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeMgr = CubeManager.getInstance(config);
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- // start job
- String jobName = getOptionValue(OPTION_JOB_NAME);
- System.out.println("Starting: " + jobName);
- job = Job.getInstance(getConf(), jobName);
-
- setJobClasspath(job);
-
- // set inputs
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- FileOutputFormat.setOutputPath(job, output);
-
- // Mapper
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(MergeCuboidMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Text.class);
-
- // Reducer - only one
- job.setReducerClass(CuboidReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
-
- // set job configuration
- job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
-
- // add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
-
- setReduceTaskNum(job, config, cubeName, 0);
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- logger.error("error in MergeCuboidJob", e);
- printUsage(options);
- throw e;
- } finally {
- if (job != null)
- cleanupTempConfFile(job.getConfiguration());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
deleted file mode 100644
index 2528e07..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.common.RowKeySplitter;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author ysong1, honma
- */
-public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
-
- private KylinConfig config;
- private String cubeName;
- private String segmentName;
- private CubeManager cubeManager;
- private CubeInstance cube;
- private CubeDesc cubeDesc;
- private CubeSegment mergedCubeSegment;
- private CubeSegment sourceCubeSegment;// Must be unique during a mapper's
- // life cycle
-
-
- // for re-encode measures that use dictionary
- private List<Pair<Integer, MeasureIngester>> dictMeasures;
- private Map<TblColRef, Dictionary<String>> oldDicts;
- private Map<TblColRef, Dictionary<String>> newDicts;
- private List<MeasureDesc> measureDescs;
- private MeasureCodec codec;
- private Object[] measureObjs;
- private ByteBuffer valueBuf;
- private Text outputValue;
-
- private Text outputKey = new Text();
-
- private byte[] newKeyBuf;
- private RowKeySplitter rowKeySplitter;
-
- private HashMap<TblColRef, Boolean> dictsNeedMerging = new HashMap<TblColRef, Boolean>();
-
- private static final Pattern JOB_NAME_PATTERN = Pattern.compile("kylin-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})");
-
- private Boolean checkNeedMerging(TblColRef col) throws IOException {
- Boolean ret = dictsNeedMerging.get(col);
- if (ret != null)
- return ret;
- else {
- ret = cubeDesc.getRowkey().isUseDictionary(col);
- if (ret) {
- String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
- ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
- }
- dictsNeedMerging.put(col, ret);
- return ret;
- }
- }
-
- private String extractJobIDFromPath(String path) {
- Matcher matcher = JOB_NAME_PATTERN.matcher(path);
- // check the first occurance
- if (matcher.find()) {
- return matcher.group(1);
- } else {
- throw new IllegalStateException("Can not extract job ID from file path : " + path);
- }
- }
-
- private CubeSegment findSegmentWithUuid(String jobID, CubeInstance cubeInstance) {
- for (CubeSegment segment : cubeInstance.getSegments()) {
- String lastBuildJobID = segment.getLastBuildJobID();
- if (lastBuildJobID != null && lastBuildJobID.equalsIgnoreCase(jobID)) {
- return segment;
- }
- }
-
- throw new IllegalStateException("No merging segment's last build job ID equals " + jobID);
-
- }
-
- @Override
- protected void setup(Context context) throws IOException, InterruptedException {
- super.publishConfiguration(context.getConfiguration());
-
- cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
- segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME).toUpperCase();
-
- config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
-
- cubeManager = CubeManager.getInstance(config);
- cube = cubeManager.getCube(cubeName);
- cubeDesc = cube.getDescriptor();
- mergedCubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
-
- // int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
- newKeyBuf = new byte[256];// size will auto-grow
-
- // decide which source segment
- InputSplit inputSplit = context.getInputSplit();
- String filePath = ((FileSplit) inputSplit).getPath().toString();
- System.out.println("filePath:" + filePath);
- String jobID = extractJobIDFromPath(filePath);
- System.out.println("jobID:" + jobID);
- sourceCubeSegment = findSegmentWithUuid(jobID, cube);
- System.out.println(sourceCubeSegment);
-
- this.rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
-
-
- measureDescs = cubeDesc.getMeasures();
- codec = new MeasureCodec(measureDescs);
- measureObjs = new Object[measureDescs.size()];
- valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- outputValue = new Text();
-
- dictMeasures = Lists.newArrayList();
- for (int i = 0; i < measureDescs.size(); i++) {
- MeasureDesc measureDesc = measureDescs.get(i);
- MeasureType measureType = measureDesc.getFunction().getMeasureType();
- if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
- dictMeasures.add(Pair.newPair(i, measureType.newIngester()));
- }
- }
- if (dictMeasures.size() > 0) {
- oldDicts = sourceCubeSegment.buildDictionaryMap();
- newDicts = mergedCubeSegment.buildDictionaryMap();
- }
- }
-
- @Override
- public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- long cuboidID = rowKeySplitter.split(key.getBytes(), key.getBytes().length);
- Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidID);
-
- SplittedBytes[] splittedByteses = rowKeySplitter.getSplitBuffers();
- int bufOffset = 0;
- BytesUtil.writeLong(cuboidID, newKeyBuf, bufOffset, RowConstants.ROWKEY_CUBOIDID_LEN);
- bufOffset += RowConstants.ROWKEY_CUBOIDID_LEN;
-
- for (int i = 0; i < cuboid.getColumns().size(); ++i) {
- TblColRef col = cuboid.getColumns().get(i);
-
- if (this.checkNeedMerging(col)) {
- // if dictionary on fact table column, needs rewrite
- DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
- Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
-
- while (sourceDict.getSizeOfValue() > newKeyBuf.length - bufOffset || mergedDict.getSizeOfValue() > newKeyBuf.length - bufOffset) {
- byte[] oldBuf = newKeyBuf;
- newKeyBuf = new byte[2 * newKeyBuf.length];
- System.arraycopy(oldBuf, 0, newKeyBuf, 0, oldBuf.length);
- }
-
- int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[i + 1].value, 0, splittedByteses[i + 1].length);
- int idInMergedDict;
-
- int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBuf, bufOffset);
- if (size < 0) {
- idInMergedDict = mergedDict.nullId();
- } else {
- idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
- }
-
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
- bufOffset += mergedDict.getSizeOfId();
- } else {
- // keep as it is
- while (splittedByteses[i + 1].length > newKeyBuf.length - bufOffset) {
- byte[] oldBuf = newKeyBuf;
- newKeyBuf = new byte[2 * newKeyBuf.length];
- System.arraycopy(oldBuf, 0, newKeyBuf, 0, oldBuf.length);
- }
-
- System.arraycopy(splittedByteses[i + 1].value, 0, newKeyBuf, bufOffset, splittedByteses[i + 1].length);
- bufOffset += splittedByteses[i + 1].length;
- }
- }
- byte[] newKey = Arrays.copyOf(newKeyBuf, bufOffset);
- outputKey.set(newKey, 0, newKey.length);
-
-
- // re-encode measures if dictionary is used
- if (dictMeasures.size() > 0) {
- codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
- for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
- int i = pair.getFirst();
- MeasureIngester ingester = pair.getSecond();
- measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
- }
- valueBuf.clear();
- codec.encode(measureObjs, valueBuf);
- outputValue.set(valueBuf.array(), 0, valueBuf.position());
- value = outputValue;
- }
-
- context.write(outputKey, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
deleted file mode 100644
index e46093c..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.dao.ExecutableDao;
-import org.apache.kylin.job.dao.ExecutableOutputPO;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-
-public class MetadataCleanupJob extends AbstractHadoopJob {
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused metadata").create("delete");
-
- protected static final Logger log = LoggerFactory.getLogger(MetadataCleanupJob.class);
-
- boolean delete = false;
-
- private KylinConfig config = null;
-
- public static final long TIME_THREADSHOLD = 2 * 24 * 3600 * 1000l; // 2 days
- public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 * 1000l; // 30 days
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
- */
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- log.info("----- jobs args: " + Arrays.toString(args));
- try {
- options.addOption(OPTION_DELETE);
- parseOptions(options, args);
-
- log.info("options: '" + getOptionsAsString() + "'");
- log.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
- delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
-
- config = KylinConfig.getInstanceFromEnv();
-
- cleanup();
-
- return 0;
- } catch (Exception e) {
- e.printStackTrace(System.err);
- throw e;
- }
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(config);
- }
-
- private boolean isOlderThanThreshold(long resourceTime) {
- long currentTime = System.currentTimeMillis();
-
- if (currentTime - resourceTime > TIME_THREADSHOLD)
- return true;
- return false;
- }
-
- public void cleanup() throws Exception {
- CubeManager cubeManager = CubeManager.getInstance(config);
-
- Set<String> activeResourceList = Sets.newHashSet();
- for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
- for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
- activeResourceList.addAll(segment.getSnapshotPaths());
- activeResourceList.addAll(segment.getDictionaryPaths());
- }
- }
-
- List<String> toDeleteResource = Lists.newArrayList();
-
- // two level resources, snapshot tables and cube statistics
- for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT }) {
- ArrayList<String> snapshotTables = getStore().listResources(resourceRoot);
-
- if (snapshotTables != null) {
- for (String snapshotTable : snapshotTables) {
- ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
- if (snapshotNames != null)
- for (String snapshot : snapshotNames) {
- if (!activeResourceList.contains(snapshot)) {
- long ts = getStore().getResourceTimestamp(snapshot);
- if (isOlderThanThreshold(ts))
- toDeleteResource.add(snapshot);
- }
- }
- }
- }
- }
-
- // three level resources, only dictionaries
- ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
-
- if (dictTables != null) {
- for (String table : dictTables) {
- ArrayList<String> tableColNames = getStore().listResources(table);
- if (tableColNames != null)
- for (String tableCol : tableColNames) {
- ArrayList<String> dictionaries = getStore().listResources(tableCol);
- if (dictionaries != null)
- for (String dict : dictionaries)
- if (!activeResourceList.contains(dict)) {
- long ts = getStore().getResourceTimestamp(dict);
- if (isOlderThanThreshold(ts))
- toDeleteResource.add(dict);
- }
- }
- }
- }
-
- // delete old and completed jobs
- ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
- List<ExecutablePO> allExecutable = executableDao.getJobs();
- for (ExecutablePO executable : allExecutable) {
- long lastModified = executable.getLastModified();
- ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
- if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (output.getStatus().equals(JobStatusEnum.FINISHED.toString()) || output.getStatus().equals(JobStatusEnum.DISCARDED.toString()))) {
- toDeleteResource.add(ResourceStore.EXECUTE_PATH_ROOT + "/" + executable.getUuid());
- toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_ROOT + "/" + executable.getUuid());
-
- for (ExecutablePO task : executable.getTasks()) {
- toDeleteResource.add(ResourceStore.EXECUTE_PATH_ROOT + "/" + task.getUuid());
- toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_ROOT + "/" + task.getUuid());
- }
- }
- }
-
- if (toDeleteResource.size() > 0) {
- logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");
-
- for (String s : toDeleteResource) {
- logger.info(s);
- if (delete == true) {
- getStore().deleteResource(s);
- }
- }
- } else {
- logger.info("No resource to be cleaned up from metadata store;");
- }
-
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new MetadataCleanupJob(), args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidJob.java
deleted file mode 100644
index feb4dc4..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidJob.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class NDCuboidJob extends CuboidJob {
-
- public NDCuboidJob() {
- this.setMapperClass(NDCuboidMapper.class);
- }
-
- public static void main(String[] args) throws Exception {
- CuboidJob job = new NDCuboidJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
deleted file mode 100644
index 8725ed8..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.common.RowKeySplitter;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
-
- private static final Logger logger = LoggerFactory.getLogger(NDCuboidMapper.class);
-
- private Text outputKey = new Text();
- private String cubeName;
- private String segmentName;
- private CubeDesc cubeDesc;
- private CuboidScheduler cuboidScheduler;
-
- private int handleCounter;
- private int skipCounter;
-
- private byte[] keyBuf = new byte[4096];
- private RowKeySplitter rowKeySplitter;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
- segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME).toUpperCase();
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
-
- CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
- CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
- cubeDesc = cube.getDescriptor();
-
- // initialize CubiodScheduler
- cuboidScheduler = new CuboidScheduler(cubeDesc);
-
- rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
- }
-
- private int buildKey(Cuboid parentCuboid, Cuboid childCuboid, SplittedBytes[] splitBuffers) {
- int offset = 0;
-
- // cuboid id
- System.arraycopy(childCuboid.getBytes(), 0, keyBuf, offset, childCuboid.getBytes().length);
- offset += childCuboid.getBytes().length;
-
- // rowkey columns
- long mask = Long.highestOneBit(parentCuboid.getId());
- long parentCuboidId = parentCuboid.getId();
- long childCuboidId = childCuboid.getId();
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
- int index = 1; // skip cuboidId
- for (int i = 0; i < parentCuboidIdActualLength; i++) {
- if ((mask & parentCuboidId) > 0) {// if the this bit position equals
- // 1
- if ((mask & childCuboidId) > 0) {// if the child cuboid has this
- // column
- System.arraycopy(splitBuffers[index].value, 0, keyBuf, offset, splitBuffers[index].length);
- offset += splitBuffers[index].length;
- }
- index++;
- }
- mask = mask >> 1;
- }
-
- return offset;
- }
-
- @Override
- public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- long cuboidId = rowKeySplitter.split(key.getBytes(), key.getLength());
- Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
-
- Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
-
- // if still empty or null
- if (myChildren == null || myChildren.size() == 0) {
- context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L);
- skipCounter++;
- if (skipCounter % BatchConstants.COUNTER_MAX == 0) {
- logger.info("Skipped " + skipCounter + " records!");
- }
- return;
- }
-
- context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L);
-
- handleCounter++;
- if (handleCounter % BatchConstants.COUNTER_MAX == 0) {
- logger.info("Handled " + handleCounter + " records!");
- }
-
- for (Long child : myChildren) {
- Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
- int keyLength = buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
- outputKey.set(keyBuf, 0, keyLength);
- context.write(outputKey, value);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
deleted file mode 100644
index 9c50122..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xjiang, ysong1
- *
- */
-
-public class RangeKeyDistributionJob extends AbstractHadoopJob {
- protected static final Logger log = LoggerFactory.getLogger(RangeKeyDistributionJob.class);
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
- */
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_CUBE_NAME);
-
- parseOptions(options, args);
-
- // start job
- String jobName = getOptionValue(OPTION_JOB_NAME);
- job = Job.getInstance(getConf(), jobName);
-
- setJobClasspath(job);
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- FileOutputFormat.setOutputPath(job, output);
- // job.getConfiguration().set("dfs.block.size", "67108864");
-
- // Mapper
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(RangeKeyDistributionMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(LongWritable.class);
-
- // Reducer - only one
- job.setReducerClass(RangeKeyDistributionReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(LongWritable.class);
- job.setNumReduceTasks(1);
-
- this.deletePath(job.getConfiguration(), output);
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- CubeInstance cube = cubeMgr.getCube(cubeName);
- DataModelDesc.RealizationCapacity cubeCapacity = cube.getDescriptor().getModel().getCapacity();
- int regionSplitSize = KylinConfig.getInstanceFromEnv().getHBaseRegionCut(cubeCapacity.toString());
- int maxRegionCount = KylinConfig.getInstanceFromEnv().getHBaseRegionCountMax();
- int minRegionCount = KylinConfig.getInstanceFromEnv().getHBaseRegionCountMin();
- job.getConfiguration().set(BatchConstants.REGION_SPLIT_SIZE, String.valueOf(regionSplitSize));
- job.getConfiguration().set(BatchConstants.REGION_NUMBER_MAX, String.valueOf(maxRegionCount));
- job.getConfiguration().set(BatchConstants.REGION_NUMBER_MIN, String.valueOf(minRegionCount));
-
- return waitForCompletion(job);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new RangeKeyDistributionJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapper.java
deleted file mode 100644
index 33baf45..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapper.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.mr.KylinMapper;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionMapper extends KylinMapper<Text, Text, Text, LongWritable> {
-
- private static final long ONE_MEGA_BYTES = 1L * 1024L * 1024L;
-
- private LongWritable outputValue = new LongWritable(0);
-
- private long bytesRead = 0;
-
- private Text lastKey;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- }
-
- @Override
- public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- lastKey = key;
-
- int bytesLength = key.getLength() + value.getLength();
- bytesRead += bytesLength;
-
- if (bytesRead >= ONE_MEGA_BYTES) {
- outputValue.set(bytesRead);
- context.write(key, outputValue);
-
- // reset bytesRead
- bytesRead = 0;
- }
-
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- if (lastKey != null) {
- outputValue.set(bytesRead);
- context.write(lastKey, outputValue);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
deleted file mode 100644
index b3ab4db..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducer.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.cube.model.v1.CubeDesc.CubeCapacity;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable, Text, LongWritable> {
-
- public static final long ONE_GIGA_BYTES = 1024L * 1024L * 1024L;
-
- private static final Logger logger = LoggerFactory.getLogger(RangeKeyDistributionReducer.class);
-
- private LongWritable outputValue = new LongWritable(0);
-
- private int minRegionCount = 1;
- private int maxRegionCount = 500;
- private int cut = 10;
- private long bytesRead = 0;
- private List<Text> gbPoints = new ArrayList<Text>();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- if (context.getConfiguration().get(BatchConstants.REGION_SPLIT_SIZE) != null) {
- cut = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_SPLIT_SIZE));
- }
-
- if (context.getConfiguration().get(BatchConstants.REGION_NUMBER_MIN) != null) {
- minRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_NUMBER_MIN));
- }
-
- if (context.getConfiguration().get(BatchConstants.REGION_NUMBER_MAX) != null) {
- maxRegionCount = Integer.valueOf(context.getConfiguration().get(BatchConstants.REGION_NUMBER_MAX));
- }
-
- logger.info("Chosen cut for htable is " + cut + ", max region count=" + maxRegionCount + ", min region count =" + minRegionCount);
- }
-
- @Override
- public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
- for (LongWritable v : values) {
- bytesRead += v.get();
- }
-
- if (bytesRead >= ONE_GIGA_BYTES) {
- gbPoints.add(new Text(key));
- bytesRead = 0; // reset bytesRead
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- int nRegion = Math.round((float) gbPoints.size() / (float) cut);
- nRegion = Math.max(minRegionCount, nRegion);
- nRegion = Math.min(maxRegionCount, nRegion);
-
- int gbPerRegion = gbPoints.size() / nRegion;
- gbPerRegion = Math.max(1, gbPerRegion);
-
- System.out.println(nRegion + " regions");
- System.out.println(gbPerRegion + " GB per region");
-
- for (int i = gbPerRegion; i < gbPoints.size(); i += gbPerRegion) {
- Text key = gbPoints.get(i);
- outputValue.set(i);
- System.out.println(StringUtils.byteToHexString(key.getBytes()) + "\t" + outputValue.get());
- context.write(key, outputValue);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
deleted file mode 100644
index 9372f18..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-
-/**
- * @author ysong1
- *
- */
-public class RowKeyDistributionCheckerJob extends AbstractHadoopJob {
-
- @SuppressWarnings("static-access")
- protected static final Option rowKeyStatsFilePath = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("rowKeyStatsFilePath").create("rowKeyStatsFilePath");
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_JOB_NAME);
- options.addOption(rowKeyStatsFilePath);
-
- parseOptions(options, args);
-
- String statsFilePath = getOptionValue(rowKeyStatsFilePath);
-
- // start job
- String jobName = getOptionValue(OPTION_JOB_NAME);
- job = Job.getInstance(getConf(), jobName);
-
- setJobClasspath(job);
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- FileOutputFormat.setOutputPath(job, output);
-
- // Mapper
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(RowKeyDistributionCheckerMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(LongWritable.class);
-
- // Reducer - only one
- job.setReducerClass(RowKeyDistributionCheckerReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(LongWritable.class);
- job.setNumReduceTasks(1);
-
- job.getConfiguration().set("rowKeyStatsFilePath", statsFilePath);
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new RowKeyDistributionCheckerJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerMapper.java
deleted file mode 100644
index 50b0499..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerMapper.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.mr.KylinMapper;
-
-/**
- * @author ysong1
- *
- */
-public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Text, LongWritable> {
-
- String rowKeyStatsFilePath;
- byte[][] splitKeys;
- Map<Text, Long> resultMap;
- List<Text> keyList;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- rowKeyStatsFilePath = context.getConfiguration().get("rowKeyStatsFilePath");
- splitKeys = this.getSplits(context.getConfiguration(), new Path(rowKeyStatsFilePath));
-
- resultMap = new HashMap<Text, Long>();
- keyList = new ArrayList<Text>();
- for (int i = 0; i < splitKeys.length; i++) {
- Text key = new Text(splitKeys[i]);
- resultMap.put(key, 0L);
- keyList.add(new Text(splitKeys[i]));
- }
- }
-
- @Override
- public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- for (Text t : keyList) {
- if (key.compareTo(t) < 0) {
- Long v = resultMap.get(t);
- long length = key.getLength() + value.getLength();
- v += length;
- resultMap.put(t, v);
- break;
- }
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- LongWritable outputValue = new LongWritable();
- for (Entry<Text, Long> kv : resultMap.entrySet()) {
- outputValue.set(kv.getValue());
- context.write(kv.getKey(), outputValue);
- }
- }
-
- @SuppressWarnings("deprecation")
- public byte[][] getSplits(Configuration conf, Path path) {
- List<byte[]> rowkeyList = new ArrayList<byte[]>();
- SequenceFile.Reader reader = null;
- try {
- reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf);
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
- while (reader.next(key, value)) {
- byte[] tmp = ((Text) key).copyBytes();
- if (rowkeyList.contains(tmp) == false) {
- rowkeyList.add(tmp);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- IOUtils.closeStream(reader);
- }
-
- byte[][] retValue = rowkeyList.toArray(new byte[rowkeyList.size()][]);
-
- return retValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerReducer.java
deleted file mode 100644
index 83e503e..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerReducer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.mr.KylinReducer;
-
-/**
- * @author ysong1
- *
- */
-public class RowKeyDistributionCheckerReducer extends KylinReducer<Text, LongWritable, Text, LongWritable> {
-
- LongWritable outputKey = new LongWritable(0L);
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- }
-
- @Override
- public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
-
- long length = 0;
- for (LongWritable v : values) {
- length += v.get();
- }
-
- outputKey.set(length);
- context.write(key, outputKey);
- }
-}
[46/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
deleted file mode 100644
index 0061e93..0000000
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.hll;
-
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-/**
- * @author xjiang
- *
- */
-public class HyperLogLogPlusTable {
-
- // threshold and bias data taken from google's bias correction data set:
- // https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
- public static final double[] thresholdData = { 10, 20, 40, 80, 220, 400, 900, 1800, 3100, 6500, 15500, 20000, 50000, 120000, 350000 };
-
- public static final double[][] rawEstimateData = {
- // precision 4
- { 11, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 77.2394, },
- // precision 5
- { 23, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716,
- 111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 157.0372, 158.5852, },
- // precision 6
- { 46, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 127.7926, 129.1952, 129.9564, 131.6454, 133.87, 134.5428, 136.2, 137.0294, 138.6278, 139.6782, 141.792, 143.3516, 144.2832, 146.0394, 147.0748, 148.4912, 150.849, 151.696, 153.5404, 154.073, 156.3714, 157.7216, 158.7328, 160.4208, 161.4184, 163.9424, 165.2772, 166.411, 168.1308, 168.769, 170.9258,
- 172.6828, 173.7502, 175.706, 176.3886, 179.0186, 180.4518, 181.927, 183.4172, 184.4114, 186.033, 188.5124, 189.5564, 191.6008, 192.4172, 193.8044, 194.997, 197.4548, 198.8948, 200.2346, 202.3086, 203.1548, 204.8842, 206.6508, 206.6772, 209.7254, 210.4752, 212.7228, 214.6614, 215.1676, 217.793, 218.0006, 219.9052, 221.66, 223.5588, 225.1636, 225.6882, 227.7126, 229.4502, 231.1978, 232.9756, 233.1654, 236.727, 238.1974, 237.7474, 241.1346, 242.3048, 244.1948, 245.3134, 246.879, 249.1204, 249.853, 252.6792, 253.857, 254.4486, 257.2362, 257.9534, 260.0286, 260.5632, 262.663, 264.723, 265.7566, 267.2566, 267.1624, 270.62, 272.8216, 273.2166, 275.2056, 276.2202, 278.3726, 280.3344, 281.9284, 283.9728, 284.1924, 286.4872, 287.587, 289.807, 291.1206, 292.769, 294.8708, 296.665, 297.1182, 299.4012, 300.6352, 302.1354, 304.1756, 306.1606, 307.3462, 308.5214, 309.4134, 310.8352, 313.9684, 315.837, 316.7796, 318.9858, },
- // precision 7
- { 92, 93.4934, 94.9758, 96.4574, 97.9718, 99.4954, 101.5302, 103.0756, 104.6374, 106.1782, 107.7888, 109.9522, 111.592, 113.2532, 114.9086, 116.5938, 118.9474, 120.6796, 122.4394, 124.2176, 125.9768, 128.4214, 130.2528, 132.0102, 133.8658, 135.7278, 138.3044, 140.1316, 142.093, 144.0032, 145.9092, 148.6306, 150.5294, 152.5756, 154.6508, 156.662, 159.552, 161.3724, 163.617, 165.5754, 167.7872, 169.8444, 172.7988, 174.8606, 177.2118, 179.3566, 181.4476, 184.5882, 186.6816, 189.0824, 191.0258, 193.6048, 196.4436, 198.7274, 200.957, 203.147, 205.4364, 208.7592, 211.3386, 213.781, 215.8028, 218.656, 221.6544, 223.996, 226.4718, 229.1544, 231.6098, 234.5956, 237.0616, 239.5758, 242.4878, 244.5244, 248.2146, 250.724, 252.8722, 255.5198, 258.0414, 261.941, 264.9048, 266.87, 269.4304, 272.028, 274.4708, 278.37, 281.0624, 283.4668, 286.5532, 289.4352, 293.2564, 295.2744, 298.2118, 300.7472, 304.1456, 307.2928, 309.7504, 312.5528, 315.979, 318.2102, 322.1834, 324.3494, 327.325,
- 330.6614, 332.903, 337.2544, 339.9042, 343.215, 345.2864, 348.0814, 352.6764, 355.301, 357.139, 360.658, 363.1732, 366.5902, 369.9538, 373.0828, 375.922, 378.9902, 382.7328, 386.4538, 388.1136, 391.2234, 394.0878, 396.708, 401.1556, 404.1852, 406.6372, 409.6822, 412.7796, 416.6078, 418.4916, 422.131, 424.5376, 428.1988, 432.211, 434.4502, 438.5282, 440.912, 444.0448, 447.7432, 450.8524, 453.7988, 456.7858, 458.8868, 463.9886, 466.5064, 468.9124, 472.6616, 475.4682, 478.582, 481.304, 485.2738, 488.6894, 490.329, 496.106, 497.6908, 501.1374, 504.5322, 506.8848, 510.3324, 513.4512, 516.179, 520.4412, 522.6066, 526.167, 528.7794, 533.379, 536.067, 538.46, 542.9116, 545.692, 547.9546, 552.493, 555.2722, 557.335, 562.449, 564.2014, 569.0738, 571.0974, 574.8564, 578.2996, 581.409, 583.9704, 585.8098, 589.6528, 594.5998, 595.958, 600.068, 603.3278, 608.2016, 609.9632, 612.864, 615.43, 620.7794, 621.272, 625.8644, 629.206, 633.219, 634.5154, 638.6102, },
- // precision 8
- { 184.2152, 187.2454, 190.2096, 193.6652, 196.6312, 199.6822, 203.249, 206.3296, 210.0038, 213.2074, 216.4612, 220.27, 223.5178, 227.4412, 230.8032, 234.1634, 238.1688, 241.6074, 245.6946, 249.2664, 252.8228, 257.0432, 260.6824, 264.9464, 268.6268, 272.2626, 276.8376, 280.4034, 284.8956, 288.8522, 292.7638, 297.3552, 301.3556, 305.7526, 309.9292, 313.8954, 318.8198, 322.7668, 327.298, 331.6688, 335.9466, 340.9746, 345.1672, 349.3474, 354.3028, 358.8912, 364.114, 368.4646, 372.9744, 378.4092, 382.6022, 387.843, 392.5684, 397.1652, 402.5426, 407.4152, 412.5388, 417.3592, 422.1366, 427.486, 432.3918, 437.5076, 442.509, 447.3834, 453.3498, 458.0668, 463.7346, 469.1228, 473.4528, 479.7, 484.644, 491.0518, 495.5774, 500.9068, 506.432, 512.1666, 517.434, 522.6644, 527.4894, 533.6312, 538.3804, 544.292, 550.5496, 556.0234, 562.8206, 566.6146, 572.4188, 579.117, 583.6762, 590.6576, 595.7864, 601.509, 607.5334, 612.9204, 619.772, 624.2924, 630.8654, 636.1836, 642.745, 649.1316,
- 655.0386, 660.0136, 666.6342, 671.6196, 678.1866, 684.4282, 689.3324, 695.4794, 702.5038, 708.129, 713.528, 720.3204, 726.463, 732.7928, 739.123, 744.7418, 751.2192, 756.5102, 762.6066, 769.0184, 775.2224, 781.4014, 787.7618, 794.1436, 798.6506, 805.6378, 811.766, 819.7514, 824.5776, 828.7322, 837.8048, 843.6302, 849.9336, 854.4798, 861.3388, 867.9894, 873.8196, 880.3136, 886.2308, 892.4588, 899.0816, 905.4076, 912.0064, 917.3878, 923.619, 929.998, 937.3482, 943.9506, 947.991, 955.1144, 962.203, 968.8222, 975.7324, 981.7826, 988.7666, 994.2648, 1000.3128, 1007.4082, 1013.7536, 1020.3376, 1026.7156, 1031.7478, 1037.4292, 1045.393, 1051.2278, 1058.3434, 1062.8726, 1071.884, 1076.806, 1082.9176, 1089.1678, 1095.5032, 1102.525, 1107.2264, 1115.315, 1120.93, 1127.252, 1134.1496, 1139.0408, 1147.5448, 1153.3296, 1158.1974, 1166.5262, 1174.3328, 1175.657, 1184.4222, 1190.9172, 1197.1292, 1204.4606, 1210.4578, 1218.8728, 1225.3336, 1226.6592, 1236.5768, 1241.363,
- 1249.4074, 1254.6566, 1260.8014, 1266.5454, 1274.5192, },
- // precision 9
- { 369, 374.8294, 381.2452, 387.6698, 394.1464, 400.2024, 406.8782, 413.6598, 420.462, 427.2826, 433.7102, 440.7416, 447.9366, 455.1046, 462.285, 469.0668, 476.306, 483.8448, 491.301, 498.9886, 506.2422, 513.8138, 521.7074, 529.7428, 537.8402, 545.1664, 553.3534, 561.594, 569.6886, 577.7876, 585.65, 594.228, 602.8036, 611.1666, 620.0818, 628.0824, 637.2574, 646.302, 655.1644, 664.0056, 672.3802, 681.7192, 690.5234, 700.2084, 708.831, 718.485, 728.1112, 737.4764, 746.76, 756.3368, 766.5538, 775.5058, 785.2646, 795.5902, 804.3818, 814.8998, 824.9532, 835.2062, 845.2798, 854.4728, 864.9582, 875.3292, 886.171, 896.781, 906.5716, 916.7048, 927.5322, 937.875, 949.3972, 958.3464, 969.7274, 980.2834, 992.1444, 1003.4264, 1013.0166, 1024.018, 1035.0438, 1046.34, 1057.6856, 1068.9836, 1079.0312, 1091.677, 1102.3188, 1113.4846, 1124.4424, 1135.739, 1147.1488, 1158.9202, 1169.406, 1181.5342, 1193.2834, 1203.8954, 1216.3286, 1226.2146, 1239.6684, 1251.9946, 1262.123, 1275.4338,
- 1285.7378, 1296.076, 1308.9692, 1320.4964, 1333.0998, 1343.9864, 1357.7754, 1368.3208, 1380.4838, 1392.7388, 1406.0758, 1416.9098, 1428.9728, 1440.9228, 1453.9292, 1462.617, 1476.05, 1490.2996, 1500.6128, 1513.7392, 1524.5174, 1536.6322, 1548.2584, 1562.3766, 1572.423, 1587.1232, 1596.5164, 1610.5938, 1622.5972, 1633.1222, 1647.7674, 1658.5044, 1671.57, 1683.7044, 1695.4142, 1708.7102, 1720.6094, 1732.6522, 1747.841, 1756.4072, 1769.9786, 1782.3276, 1797.5216, 1808.3186, 1819.0694, 1834.354, 1844.575, 1856.2808, 1871.1288, 1880.7852, 1893.9622, 1906.3418, 1920.6548, 1932.9302, 1945.8584, 1955.473, 1968.8248, 1980.6446, 1995.9598, 2008.349, 2019.8556, 2033.0334, 2044.0206, 2059.3956, 2069.9174, 2082.6084, 2093.7036, 2106.6108, 2118.9124, 2132.301, 2144.7628, 2159.8422, 2171.0212, 2183.101, 2193.5112, 2208.052, 2221.3194, 2233.3282, 2247.295, 2257.7222, 2273.342, 2286.5638, 2299.6786, 2310.8114, 2322.3312, 2335.516, 2349.874, 2363.5968, 2373.865, 2387.1918,
- 2401.8328, 2414.8496, 2424.544, 2436.7592, 2447.1682, 2464.1958, 2474.3438, 2489.0006, 2497.4526, 2513.6586, 2527.19, 2540.7028, 2553.768, },
- // precision 10
- { 738.1256, 750.4234, 763.1064, 775.4732, 788.4636, 801.0644, 814.488, 827.9654, 841.0832, 854.7864, 868.1992, 882.2176, 896.5228, 910.1716, 924.7752, 938.899, 953.6126, 968.6492, 982.9474, 998.5214, 1013.1064, 1028.6364, 1044.2468, 1059.4588, 1075.3832, 1091.0584, 1106.8606, 1123.3868, 1139.5062, 1156.1862, 1172.463, 1189.339, 1206.1936, 1223.1292, 1240.1854, 1257.2908, 1275.3324, 1292.8518, 1310.5204, 1328.4854, 1345.9318, 1364.552, 1381.4658, 1400.4256, 1419.849, 1438.152, 1456.8956, 1474.8792, 1494.118, 1513.62, 1532.5132, 1551.9322, 1570.7726, 1590.6086, 1610.5332, 1630.5918, 1650.4294, 1669.7662, 1690.4106, 1710.7338, 1730.9012, 1750.4486, 1770.1556, 1791.6338, 1812.7312, 1833.6264, 1853.9526, 1874.8742, 1896.8326, 1918.1966, 1939.5594, 1961.07, 1983.037, 2003.1804, 2026.071, 2047.4884, 2070.0848, 2091.2944, 2114.333, 2135.9626, 2158.2902, 2181.0814, 2202.0334, 2224.4832, 2246.39, 2269.7202, 2292.1714, 2314.2358, 2338.9346, 2360.891, 2384.0264, 2408.3834, 2430.1544
,
- 2454.8684, 2476.9896, 2501.4368, 2522.8702, 2548.0408, 2570.6738, 2593.5208, 2617.0158, 2640.2302, 2664.0962, 2687.4986, 2714.2588, 2735.3914, 2759.6244, 2781.8378, 2808.0072, 2830.6516, 2856.2454, 2877.2136, 2903.4546, 2926.785, 2951.2294, 2976.468, 3000.867, 3023.6508, 3049.91, 3073.5984, 3098.162, 3121.5564, 3146.2328, 3170.9484, 3195.5902, 3221.3346, 3242.7032, 3271.6112, 3296.5546, 3317.7376, 3345.072, 3369.9518, 3394.326, 3418.1818, 3444.6926, 3469.086, 3494.2754, 3517.8698, 3544.248, 3565.3768, 3588.7234, 3616.979, 3643.7504, 3668.6812, 3695.72, 3719.7392, 3742.6224, 3770.4456, 3795.6602, 3819.9058, 3844.002, 3869.517, 3895.6824, 3920.8622, 3947.1364, 3973.985, 3995.4772, 4021.62, 4046.628, 4074.65, 4096.2256, 4121.831, 4146.6406, 4173.276, 4195.0744, 4223.9696, 4251.3708, 4272.9966, 4300.8046, 4326.302, 4353.1248, 4374.312, 4403.0322, 4426.819, 4450.0598, 4478.5206, 4504.8116, 4528.8928, 4553.9584, 4578.8712, 4603.8384, 4632.3872, 4655.5128, 4675.821,
- 4704.6222, 4731.9862, 4755.4174, 4781.2628, 4804.332, 4832.3048, 4862.8752, 4883.4148, 4906.9544, 4935.3516, 4954.3532, 4984.0248, 5011.217, 5035.3258, 5057.3672, 5084.1828, },
- // precision 11
- { 1477, 1501.6014, 1526.5802, 1551.7942, 1577.3042, 1603.2062, 1629.8402, 1656.2292, 1682.9462, 1709.9926, 1737.3026, 1765.4252, 1793.0578, 1821.6092, 1849.626, 1878.5568, 1908.527, 1937.5154, 1967.1874, 1997.3878, 2027.37, 2058.1972, 2089.5728, 2120.1012, 2151.9668, 2183.292, 2216.0772, 2247.8578, 2280.6562, 2313.041, 2345.714, 2380.3112, 2414.1806, 2447.9854, 2481.656, 2516.346, 2551.5154, 2586.8378, 2621.7448, 2656.6722, 2693.5722, 2729.1462, 2765.4124, 2802.8728, 2838.898, 2876.408, 2913.4926, 2951.4938, 2989.6776, 3026.282, 3065.7704, 3104.1012, 3143.7388, 3181.6876, 3221.1872, 3261.5048, 3300.0214, 3339.806, 3381.409, 3421.4144, 3461.4294, 3502.2286, 3544.651, 3586.6156, 3627.337, 3670.083, 3711.1538, 3753.5094, 3797.01, 3838.6686, 3882.1678, 3922.8116, 3967.9978, 4009.9204, 4054.3286, 4097.5706, 4140.6014, 4185.544, 4229.5976, 4274.583, 4316.9438, 4361.672, 4406.2786, 4451.8628, 4496.1834, 4543.505, 4589.1816, 4632.5188, 4678.2294, 4724.8908, 4769.0194, 4817.052,
- 4861.4588, 4910.1596, 4956.4344, 5002.5238, 5048.13, 5093.6374, 5142.8162, 5187.7894, 5237.3984, 5285.6078, 5331.0858, 5379.1036, 5428.6258, 5474.6018, 5522.7618, 5571.5822, 5618.59, 5667.9992, 5714.88, 5763.454, 5808.6982, 5860.3644, 5910.2914, 5953.571, 6005.9232, 6055.1914, 6104.5882, 6154.5702, 6199.7036, 6251.1764, 6298.7596, 6350.0302, 6398.061, 6448.4694, 6495.933, 6548.0474, 6597.7166, 6646.9416, 6695.9208, 6742.6328, 6793.5276, 6842.1934, 6894.2372, 6945.3864, 6996.9228, 7044.2372, 7094.1374, 7142.2272, 7192.2942, 7238.8338, 7288.9006, 7344.0908, 7394.8544, 7443.5176, 7490.4148, 7542.9314, 7595.6738, 7641.9878, 7694.3688, 7743.0448, 7797.522, 7845.53, 7899.594, 7950.3132, 7996.455, 8050.9442, 8092.9114, 8153.1374, 8197.4472, 8252.8278, 8301.8728, 8348.6776, 8401.4698, 8453.551, 8504.6598, 8553.8944, 8604.1276, 8657.6514, 8710.3062, 8758.908, 8807.8706, 8862.1702, 8910.4668, 8960.77, 9007.2766, 9063.164, 9121.0534, 9164.1354, 9218.1594, 9267.767,
- 9319.0594, 9372.155, 9419.7126, 9474.3722, 9520.1338, 9572.368, 9622.7702, 9675.8448, 9726.5396, 9778.7378, 9827.6554, 9878.1922, 9928.7782, 9978.3984, 10026.578, 10076.5626, 10137.1618, 10177.5244, 10229.9176, },
- // precision 12
- { 2954, 3003.4782, 3053.3568, 3104.3666, 3155.324, 3206.9598, 3259.648, 3312.539, 3366.1474, 3420.2576, 3474.8376, 3530.6076, 3586.451, 3643.38, 3700.4104, 3757.5638, 3815.9676, 3875.193, 3934.838, 3994.8548, 4055.018, 4117.1742, 4178.4482, 4241.1294, 4304.4776, 4367.4044, 4431.8724, 4496.3732, 4561.4304, 4627.5326, 4693.949, 4761.5532, 4828.7256, 4897.6182, 4965.5186, 5034.4528, 5104.865, 5174.7164, 5244.6828, 5316.6708, 5387.8312, 5459.9036, 5532.476, 5604.8652, 5679.6718, 5753.757, 5830.2072, 5905.2828, 5980.0434, 6056.6264, 6134.3192, 6211.5746, 6290.0816, 6367.1176, 6447.9796, 6526.5576, 6606.1858, 6686.9144, 6766.1142, 6847.0818, 6927.9664, 7010.9096, 7091.0816, 7175.3962, 7260.3454, 7344.018, 7426.4214, 7511.3106, 7596.0686, 7679.8094, 7765.818, 7852.4248, 7936.834, 8022.363, 8109.5066, 8200.4554, 8288.5832, 8373.366, 8463.4808, 8549.7682, 8642.0522, 8728.3288, 8820.9528, 8907.727, 9001.0794, 9091.2522, 9179.988, 9269.852, 9362.6394, 9453.642, 9546.9024, 9640.6616
,
- 9732.6622, 9824.3254, 9917.7484, 10007.9392, 10106.7508, 10196.2152, 10289.8114, 10383.5494, 10482.3064, 10576.8734, 10668.7872, 10764.7156, 10862.0196, 10952.793, 11049.9748, 11146.0702, 11241.4492, 11339.2772, 11434.2336, 11530.741, 11627.6136, 11726.311, 11821.5964, 11918.837, 12015.3724, 12113.0162, 12213.0424, 12306.9804, 12408.4518, 12504.8968, 12604.586, 12700.9332, 12798.705, 12898.5142, 12997.0488, 13094.788, 13198.475, 13292.7764, 13392.9698, 13486.8574, 13590.1616, 13686.5838, 13783.6264, 13887.2638, 13992.0978, 14081.0844, 14189.9956, 14280.0912, 14382.4956, 14486.4384, 14588.1082, 14686.2392, 14782.276, 14888.0284, 14985.1864, 15088.8596, 15187.0998, 15285.027, 15383.6694, 15495.8266, 15591.3736, 15694.2008, 15790.3246, 15898.4116, 15997.4522, 16095.5014, 16198.8514, 16291.7492, 16402.6424, 16499.1266, 16606.2436, 16697.7186, 16796.3946, 16902.3376, 17005.7672, 17100.814, 17206.8282, 17305.8262, 17416.0744, 17508.4092, 17617.0178, 17715.4554,
- 17816.758, 17920.1748, 18012.9236, 18119.7984, 18223.2248, 18324.2482, 18426.6276, 18525.0932, 18629.8976, 18733.2588, 18831.0466, 18940.1366, 19032.2696, 19131.729, 19243.4864, 19349.6932, 19442.866, 19547.9448, 19653.2798, 19754.4034, 19854.0692, 19965.1224, 20065.1774, 20158.2212, 20253.353, 20366.3264, 20463.22, },
- // precision 13
- { 5908.5052, 6007.2672, 6107.347, 6208.5794, 6311.2622, 6414.5514, 6519.3376, 6625.6952, 6732.5988, 6841.3552, 6950.5972, 7061.3082, 7173.5646, 7287.109, 7401.8216, 7516.4344, 7633.3802, 7751.2962, 7870.3784, 7990.292, 8110.79, 8233.4574, 8356.6036, 8482.2712, 8607.7708, 8735.099, 8863.1858, 8993.4746, 9123.8496, 9255.6794, 9388.5448, 9522.7516, 9657.3106, 9792.6094, 9930.5642, 10068.794, 10206.7256, 10347.81, 10490.3196, 10632.0778, 10775.9916, 10920.4662, 11066.124, 11213.073, 11358.0362, 11508.1006, 11659.1716, 11808.7514, 11959.4884, 12112.1314, 12265.037, 12420.3756, 12578.933, 12734.311, 12890.0006, 13047.2144, 13207.3096, 13368.5144, 13528.024, 13689.847, 13852.7528, 14018.3168, 14180.5372, 14346.9668, 14513.5074, 14677.867, 14846.2186, 15017.4186, 15184.9716, 15356.339, 15529.2972, 15697.3578, 15871.8686, 16042.187, 16216.4094, 16389.4188, 16565.9126, 16742.3272, 16919.0042, 17094.7592, 17273.965, 17451.8342, 17634.4254, 17810.5984, 17988.9242, 18171.051,
- 18354.7938, 18539.466, 18721.0408, 18904.9972, 19081.867, 19271.9118, 19451.8694, 19637.9816, 19821.2922, 20013.1292, 20199.3858, 20387.8726, 20572.9514, 20770.7764, 20955.1714, 21144.751, 21329.9952, 21520.709, 21712.7016, 21906.3868, 22096.2626, 22286.0524, 22475.051, 22665.5098, 22862.8492, 23055.5294, 23249.6138, 23437.848, 23636.273, 23826.093, 24020.3296, 24213.3896, 24411.7392, 24602.9614, 24805.7952, 24998.1552, 25193.9588, 25389.0166, 25585.8392, 25780.6976, 25981.2728, 26175.977, 26376.5252, 26570.1964, 26773.387, 26962.9812, 27163.0586, 27368.164, 27565.0534, 27758.7428, 27961.1276, 28163.2324, 28362.3816, 28565.7668, 28758.644, 28956.9768, 29163.4722, 29354.7026, 29561.1186, 29767.9948, 29959.9986, 30164.0492, 30366.9818, 30562.5338, 30762.9928, 30976.1592, 31166.274, 31376.722, 31570.3734, 31770.809, 31974.8934, 32179.5286, 32387.5442, 32582.3504, 32794.076, 32989.9528, 33191.842, 33392.4684, 33595.659, 33801.8672, 34000.3414, 34200.0922, 34402.6792,
- 34610.0638, 34804.0084, 35011.13, 35218.669, 35418.6634, 35619.0792, 35830.6534, 36028.4966, 36229.7902, 36438.6422, 36630.7764, 36833.3102, 37048.6728, 37247.3916, 37453.5904, 37669.3614, 37854.5526, 38059.305, 38268.0936, 38470.2516, 38674.7064, 38876.167, 39068.3794, 39281.9144, 39492.8566, 39684.8628, 39898.4108, 40093.1836, 40297.6858, 40489.7086, 40717.2424, },
- // precision 14
- { 11817.475, 12015.0046, 12215.3792, 12417.7504, 12623.1814, 12830.0086, 13040.0072, 13252.503, 13466.178, 13683.2738, 13902.0344, 14123.9798, 14347.394, 14573.7784, 14802.6894, 15033.6824, 15266.9134, 15502.8624, 15741.4944, 15980.7956, 16223.8916, 16468.6316, 16715.733, 16965.5726, 17217.204, 17470.666, 17727.8516, 17986.7886, 18247.6902, 18510.9632, 18775.304, 19044.7486, 19314.4408, 19587.202, 19862.2576, 20135.924, 20417.0324, 20697.9788, 20979.6112, 21265.0274, 21550.723, 21841.6906, 22132.162, 22428.1406, 22722.127, 23020.5606, 23319.7394, 23620.4014, 23925.2728, 24226.9224, 24535.581, 24845.505, 25155.9618, 25470.3828, 25785.9702, 26103.7764, 26420.4132, 26742.0186, 27062.8852, 27388.415, 27714.6024, 28042.296, 28365.4494, 28701.1526, 29031.8008, 29364.2156, 29704.497, 30037.1458, 30380.111, 30723.8168, 31059.5114, 31404.9498, 31751.6752, 32095.2686, 32444.7792, 32794.767, 33145.204, 33498.4226, 33847.6502, 34209.006, 34560.849, 34919.4838, 35274.9778, 35635.1322
,
- 35996.3266, 36359.1394, 36722.8266, 37082.8516, 37447.7354, 37815.9606, 38191.0692, 38559.4106, 38924.8112, 39294.6726, 39663.973, 40042.261, 40416.2036, 40779.2036, 41161.6436, 41540.9014, 41921.1998, 42294.7698, 42678.5264, 43061.3464, 43432.375, 43818.432, 44198.6598, 44583.0138, 44970.4794, 45353.924, 45729.858, 46118.2224, 46511.5724, 46900.7386, 47280.6964, 47668.1472, 48055.6796, 48446.9436, 48838.7146, 49217.7296, 49613.7796, 50010.7508, 50410.0208, 50793.7886, 51190.2456, 51583.1882, 51971.0796, 52376.5338, 52763.319, 53165.5534, 53556.5594, 53948.2702, 54346.352, 54748.7914, 55138.577, 55543.4824, 55941.1748, 56333.7746, 56745.1552, 57142.7944, 57545.2236, 57935.9956, 58348.5268, 58737.5474, 59158.5962, 59542.6896, 59958.8004, 60349.3788, 60755.0212, 61147.6144, 61548.194, 61946.0696, 62348.6042, 62763.603, 63162.781, 63560.635, 63974.3482, 64366.4908, 64771.5876, 65176.7346, 65597.3916, 65995.915, 66394.0384, 66822.9396, 67203.6336, 67612.2032,
- 68019.0078, 68420.0388, 68821.22, 69235.8388, 69640.0724, 70055.155, 70466.357, 70863.4266, 71276.2482, 71677.0306, 72080.2006, 72493.0214, 72893.5952, 73314.5856, 73714.9852, 74125.3022, 74521.2122, 74933.6814, 75341.5904, 75743.0244, 76166.0278, 76572.1322, 76973.1028, 77381.6284, 77800.6092, 78189.328, 78607.0962, 79012.2508, 79407.8358, 79825.725, 80238.701, 80646.891, 81035.6436, 81460.0448, 81876.3884, },
- // precision 15
- { 23635.0036, 24030.8034, 24431.4744, 24837.1524, 25246.7928, 25661.326, 26081.3532, 26505.2806, 26933.9892, 27367.7098, 27805.318, 28248.799, 28696.4382, 29148.8244, 29605.5138, 30066.8668, 30534.2344, 31006.32, 31480.778, 31962.2418, 32447.3324, 32938.0232, 33432.731, 33930.728, 34433.9896, 34944.1402, 35457.5588, 35974.5958, 36497.3296, 37021.9096, 37554.326, 38088.0826, 38628.8816, 39171.3192, 39723.2326, 40274.5554, 40832.3142, 41390.613, 41959.5908, 42532.5466, 43102.0344, 43683.5072, 44266.694, 44851.2822, 45440.7862, 46038.0586, 46640.3164, 47241.064, 47846.155, 48454.7396, 49076.9168, 49692.542, 50317.4778, 50939.65, 51572.5596, 52210.2906, 52843.7396, 53481.3996, 54127.236, 54770.406, 55422.6598, 56078.7958, 56736.7174, 57397.6784, 58064.5784, 58730.308, 59404.9784, 60077.0864, 60751.9158, 61444.1386, 62115.817, 62808.7742, 63501.4774, 64187.5454, 64883.6622, 65582.7468, 66274.5318, 66976.9276, 67688.7764, 68402.138, 69109.6274, 69822.9706, 70543.6108,
- 71265.5202, 71983.3848, 72708.4656, 73433.384, 74158.4664, 74896.4868, 75620.9564, 76362.1434, 77098.3204, 77835.7662, 78582.6114, 79323.9902, 80067.8658, 80814.9246, 81567.0136, 82310.8536, 83061.9952, 83821.4096, 84580.8608, 85335.547, 86092.5802, 86851.6506, 87612.311, 88381.2016, 89146.3296, 89907.8974, 90676.846, 91451.4152, 92224.5518, 92995.8686, 93763.5066, 94551.2796, 95315.1944, 96096.1806, 96881.0918, 97665.679, 98442.68, 99229.3002, 100011.0994, 100790.6386, 101580.1564, 102377.7484, 103152.1392, 103944.2712, 104730.216, 105528.6336, 106324.9398, 107117.6706, 107890.3988, 108695.2266, 109485.238, 110294.7876, 111075.0958, 111878.0496, 112695.2864, 113464.5486, 114270.0474, 115068.608, 115884.3626, 116673.2588, 117483.3716, 118275.097, 119085.4092, 119879.2808, 120687.5868, 121499.9944, 122284.916, 123095.9254, 123912.5038, 124709.0454, 125503.7182, 126323.259, 127138.9412, 127943.8294, 128755.646, 129556.5354, 130375.3298, 131161.4734, 131971.1962,
- 132787.5458, 133588.1056, 134431.351, 135220.2906, 136023.398, 136846.6558, 137667.0004, 138463.663, 139283.7154, 140074.6146, 140901.3072, 141721.8548, 142543.2322, 143356.1096, 144173.7412, 144973.0948, 145794.3162, 146609.5714, 147420.003, 148237.9784, 149050.5696, 149854.761, 150663.1966, 151494.0754, 152313.1416, 153112.6902, 153935.7206, 154746.9262, 155559.547, 156401.9746, 157228.7036, 158008.7254, 158820.75, 159646.9184, 160470.4458, 161279.5348, 162093.3114, 162918.542, 163729.2842, },
- // precision 16
- { 47271, 48062.3584, 48862.7074, 49673.152, 50492.8416, 51322.9514, 52161.03, 53009.407, 53867.6348, 54734.206, 55610.5144, 56496.2096, 57390.795, 58297.268, 59210.6448, 60134.665, 61068.0248, 62010.4472, 62962.5204, 63923.5742, 64895.0194, 65876.4182, 66862.6136, 67862.6968, 68868.8908, 69882.8544, 70911.271, 71944.0924, 72990.0326, 74040.692, 75100.6336, 76174.7826, 77252.5998, 78340.2974, 79438.2572, 80545.4976, 81657.2796, 82784.6336, 83915.515, 85059.7362, 86205.9368, 87364.4424, 88530.3358, 89707.3744, 90885.9638, 92080.197, 93275.5738, 94479.391, 95695.918, 96919.2236, 98148.4602, 99382.3474, 100625.6974, 101878.0284, 103141.6278, 104409.4588, 105686.2882, 106967.5402, 108261.6032, 109548.1578, 110852.0728, 112162.231, 113479.0072, 114806.2626, 116137.9072, 117469.5048, 118813.5186, 120165.4876, 121516.2556, 122875.766, 124250.5444, 125621.2222, 127003.2352, 128387.848, 129775.2644, 131181.7776, 132577.3086, 133979.9458, 135394.1132, 136800.9078, 138233.217,
- 139668.5308, 141085.212, 142535.2122, 143969.0684, 145420.2872, 146878.1542, 148332.7572, 149800.3202, 151269.66, 152743.6104, 154213.0948, 155690.288, 157169.4246, 158672.1756, 160160.059, 161650.6854, 163145.7772, 164645.6726, 166159.1952, 167682.1578, 169177.3328, 170700.0118, 172228.8964, 173732.6664, 175265.5556, 176787.799, 178317.111, 179856.6914, 181400.865, 182943.4612, 184486.742, 186033.4698, 187583.7886, 189148.1868, 190688.4526, 192250.1926, 193810.9042, 195354.2972, 196938.7682, 198493.5898, 200079.2824, 201618.912, 203205.5492, 204765.5798, 206356.1124, 207929.3064, 209498.7196, 211086.229, 212675.1324, 214256.7892, 215826.2392, 217412.8474, 218995.6724, 220618.6038, 222207.1166, 223781.0364, 225387.4332, 227005.7928, 228590.4336, 230217.8738, 231805.1054, 233408.9, 234995.3432, 236601.4956, 238190.7904, 239817.2548, 241411.2832, 243002.4066, 244640.1884, 246255.3128, 247849.3508, 249479.9734, 251106.8822, 252705.027, 254332.9242, 255935.129,
- 257526.9014, 259154.772, 260777.625, 262390.253, 264004.4906, 265643.59, 267255.4076, 268873.426, 270470.7252, 272106.4804, 273722.4456, 275337.794, 276945.7038, 278592.9154, 280204.3726, 281841.1606, 283489.171, 285130.1716, 286735.3362, 288364.7164, 289961.1814, 291595.5524, 293285.683, 294899.6668, 296499.3434, 298128.0462, 299761.8946, 301394.2424, 302997.6748, 304615.1478, 306269.7724, 307886.114, 309543.1028, 311153.2862, 312782.8546, 314421.2008, 316033.2438, 317692.9636, 319305.2648, 320948.7406, 322566.3364, 324228.4224, 325847.1542, },
- // precision 17
- { 94542, 96125.811, 97728.019, 99348.558, 100987.9705, 102646.7565, 104324.5125, 106021.7435, 107736.7865, 109469.272, 111223.9465, 112995.219, 114787.432, 116593.152, 118422.71, 120267.2345, 122134.6765, 124020.937, 125927.2705, 127851.255, 129788.9485, 131751.016, 133726.8225, 135722.592, 137736.789, 139770.568, 141821.518, 143891.343, 145982.1415, 148095.387, 150207.526, 152355.649, 154515.6415, 156696.05, 158887.7575, 161098.159, 163329.852, 165569.053, 167837.4005, 170121.6165, 172420.4595, 174732.6265, 177062.77, 179412.502, 181774.035, 184151.939, 186551.6895, 188965.691, 191402.8095, 193857.949, 196305.0775, 198774.6715, 201271.2585, 203764.78, 206299.3695, 208818.1365, 211373.115, 213946.7465, 216532.076, 219105.541, 221714.5375, 224337.5135, 226977.5125, 229613.0655, 232270.2685, 234952.2065, 237645.3555, 240331.1925, 243034.517, 245756.0725, 248517.6865, 251232.737, 254011.3955, 256785.995, 259556.44, 262368.335, 265156.911, 267965.266, 270785.583, 273616.0495
,
- 276487.4835, 279346.639, 282202.509, 285074.3885, 287942.2855, 290856.018, 293774.0345, 296678.5145, 299603.6355, 302552.6575, 305492.9785, 308466.8605, 311392.581, 314347.538, 317319.4295, 320285.9785, 323301.7325, 326298.3235, 329301.3105, 332301.987, 335309.791, 338370.762, 341382.923, 344431.1265, 347464.1545, 350507.28, 353619.2345, 356631.2005, 359685.203, 362776.7845, 365886.488, 368958.2255, 372060.6825, 375165.4335, 378237.935, 381328.311, 384430.5225, 387576.425, 390683.242, 393839.648, 396977.8425, 400101.9805, 403271.296, 406409.8425, 409529.5485, 412678.7, 415847.423, 419020.8035, 422157.081, 425337.749, 428479.6165, 431700.902, 434893.1915, 438049.582, 441210.5415, 444379.2545, 447577.356, 450741.931, 453959.548, 457137.0935, 460329.846, 463537.4815, 466732.3345, 469960.5615, 473164.681, 476347.6345, 479496.173, 482813.1645, 486025.6995, 489249.4885, 492460.1945, 495675.8805, 498908.0075, 502131.802, 505374.3855, 508550.9915, 511806.7305, 515026.776
,
- 518217.0005, 521523.9855, 524705.9855, 527950.997, 531210.0265, 534472.497, 537750.7315, 540926.922, 544207.094, 547429.4345, 550666.3745, 553975.3475, 557150.7185, 560399.6165, 563662.697, 566916.7395, 570146.1215, 573447.425, 576689.6245, 579874.5745, 583202.337, 586503.0255, 589715.635, 592910.161, 596214.3885, 599488.035, 602740.92, 605983.0685, 609248.67, 612491.3605, 615787.912, 619107.5245, 622307.9555, 625577.333, 628840.4385, 632085.2155, 635317.6135, 638691.7195, 641887.467, 645139.9405, 648441.546, 651666.252, 654941.845, },
- // precision 18
- { 189084, 192250.913, 195456.774, 198696.946, 201977.762, 205294.444, 208651.754, 212042.099, 215472.269, 218941.91, 222443.912, 225996.845, 229568.199, 233193.568, 236844.457, 240543.233, 244279.475, 248044.27, 251854.588, 255693.2, 259583.619, 263494.621, 267445.385, 271454.061, 275468.769, 279549.456, 283646.446, 287788.198, 291966.099, 296181.164, 300431.469, 304718.618, 309024.004, 313393.508, 317760.803, 322209.731, 326675.061, 331160.627, 335654.47, 340241.442, 344841.833, 349467.132, 354130.629, 358819.432, 363574.626, 368296.587, 373118.482, 377914.93, 382782.301, 387680.669, 392601.981, 397544.323, 402529.115, 407546.018, 412593.658, 417638.657, 422762.865, 427886.169, 433017.167, 438213.273, 443441.254, 448692.421, 453937.533, 459239.049, 464529.569, 469910.083, 475274.03, 480684.473, 486070.26, 491515.237, 496995.651, 502476.617, 507973.609, 513497.19, 519083.233, 524726.509, 530305.505, 535945.728, 541584.404, 547274.055, 552967.236, 558667.862, 564360.216,
- 570128.148, 575965.08, 581701.952, 587532.523, 593361.144, 599246.128, 605033.418, 610958.779, 616837.117, 622772.818, 628672.04, 634675.369, 640574.831, 646585.739, 652574.547, 658611.217, 664642.684, 670713.914, 676737.681, 682797.313, 688837.897, 694917.874, 701009.882, 707173.648, 713257.254, 719415.392, 725636.761, 731710.697, 737906.209, 744103.074, 750313.39, 756504.185, 762712.579, 768876.985, 775167.859, 781359, 787615.959, 793863.597, 800245.477, 806464.582, 812785.294, 819005.925, 825403.057, 831676.197, 837936.284, 844266.968, 850642.711, 856959.756, 863322.774, 869699.931, 876102.478, 882355.787, 888694.463, 895159.952, 901536.143, 907872.631, 914293.672, 920615.14, 927130.974, 933409.404, 939922.178, 946331.47, 952745.93, 959209.264, 965590.224, 972077.284, 978501.961, 984953.19, 991413.271, 997817.479, 1004222.658, 1010725.676, 1017177.138, 1023612.529, 1030098.236, 1036493.719, 1043112.207, 1049537.036, 1056008.096, 1062476.184, 1068942.337,
- 1075524.95, 1081932.864, 1088426.025, 1094776.005, 1101327.448, 1107901.673, 1114423.639, 1120884.602, 1127324.923, 1133794.24, 1140328.886, 1146849.376, 1153346.682, 1159836.502, 1166478.703, 1172953.304, 1179391.502, 1185950.982, 1192544.052, 1198913.41, 1205430.994, 1212015.525, 1218674.042, 1225121.683, 1231551.101, 1238126.379, 1244673.795, 1251260.649, 1257697.86, 1264320.983, 1270736.319, 1277274.694, 1283804.95, 1290211.514, 1296858.568, 1303455.691, } };
-
- public static final double[][] biasData = {
- // precision 4
- { 10, 9.717, 9.207, 8.7896, 8.2882, 7.8204, 7.3772, 6.9342, 6.5202, 6.161, 5.7722, 5.4636, 5.0396, 4.6766, 4.3566, 4.0454, 3.7936, 3.4856, 3.2666, 2.9946, 2.766, 2.4692, 2.3638, 2.0764, 1.7864, 1.7602, 1.4814, 1.433, 1.2926, 1.0664, 0.999600000000001, 0.7956, 0.5366, 0.589399999999998, 0.573799999999999, 0.269799999999996, 0.368200000000002, 0.0544000000000011, 0.234200000000001, 0.0108000000000033, -0.203400000000002, -0.0701999999999998, -0.129600000000003, -0.364199999999997, -0.480600000000003, -0.226999999999997, -0.322800000000001, -0.382599999999996, -0.511200000000002, -0.669600000000003, -0.749400000000001, -0.500399999999999, -0.617600000000003, -0.6922, -0.601599999999998, -0.416200000000003, -0.338200000000001, -0.782600000000002, -0.648600000000002, -0.919800000000002, -0.851799999999997, -0.962400000000002, -0.6402, -1.1922, -1.0256, -1.086, -1.21899999999999, -0.819400000000002, -0.940600000000003, -1.1554, -1.2072, -1.1752, -1.16759999999999,
- -1.14019999999999, -1.3754, -1.29859999999999, -1.607, -1.3292, -1.7606, },
- // precision 5
- { 22, 21.1194, 20.8208, 20.2318, 19.77, 19.2436, 18.7774, 18.2848, 17.8224, 17.3742, 16.9336, 16.503, 16.0494, 15.6292, 15.2124, 14.798, 14.367, 13.9728, 13.5944, 13.217, 12.8438, 12.3696, 12.0956, 11.7044, 11.324, 11.0668, 10.6698, 10.3644, 10.049, 9.6918, 9.4146, 9.082, 8.687, 8.5398, 8.2462, 7.857, 7.6606, 7.4168, 7.1248, 6.9222, 6.6804, 6.447, 6.3454, 5.9594, 5.7636, 5.5776, 5.331, 5.19, 4.9676, 4.7564, 4.5314, 4.4442, 4.3708, 3.9774, 3.9624, 3.8796, 3.755, 3.472, 3.2076, 3.1024, 2.8908, 2.7338, 2.7728, 2.629, 2.413, 2.3266, 2.1524, 2.2642, 2.1806, 2.0566, 1.9192, 1.7598, 1.3516, 1.5802, 1.43859999999999, 1.49160000000001, 1.1524, 1.1892, 0.841399999999993, 0.879800000000003, 0.837599999999995, 0.469800000000006, 0.765600000000006, 0.331000000000003, 0.591399999999993, 0.601200000000006, 0.701599999999999, 0.558199999999999, 0.339399999999998, 0.354399999999998, 0.491200000000006, 0.308000000000007, 0.355199999999996, -0.0254000000000048, 0.205200000000005,
- -0.272999999999996, 0.132199999999997, 0.394400000000005, -0.241200000000006, 0.242000000000004, 0.191400000000002, 0.253799999999998, -0.122399999999999, -0.370800000000003, 0.193200000000004, -0.0848000000000013, 0.0867999999999967, -0.327200000000005, -0.285600000000002, 0.311400000000006, -0.128399999999999, -0.754999999999995, -0.209199999999996, -0.293599999999998, -0.364000000000004, -0.253600000000006, -0.821200000000005, -0.253600000000006, -0.510400000000004, -0.383399999999995, -0.491799999999998, -0.220200000000006, -0.0972000000000008, -0.557400000000001, -0.114599999999996, -0.295000000000002, -0.534800000000004, 0.346399999999988, -0.65379999999999, 0.0398000000000138, 0.0341999999999985, -0.995800000000003, -0.523400000000009, -0.489000000000004, -0.274799999999999, -0.574999999999989, -0.482799999999997, 0.0571999999999946, -0.330600000000004, -0.628800000000012, -0.140199999999993, -0.540600000000012, -0.445999999999998, -0.599400000000003,
- -0.262599999999992, 0.163399999999996, -0.100599999999986, -0.39500000000001, -1.06960000000001, -0.836399999999998, -0.753199999999993, -0.412399999999991, -0.790400000000005, -0.29679999999999, -0.28540000000001, -0.193000000000012, -0.0772000000000048, -0.962799999999987, -0.414800000000014, },
- // precision 6
- { 45, 44.1902, 43.271, 42.8358, 41.8142, 41.2854, 40.317, 39.354, 38.8924, 37.9436, 37.4596, 36.5262, 35.6248, 35.1574, 34.2822, 33.837, 32.9636, 32.074, 31.7042, 30.7976, 30.4772, 29.6564, 28.7942, 28.5004, 27.686, 27.291, 26.5672, 25.8556, 25.4982, 24.8204, 24.4252, 23.7744, 23.0786, 22.8344, 22.0294, 21.8098, 21.0794, 20.5732, 20.1878, 19.5648, 19.2902, 18.6784, 18.3352, 17.8946, 17.3712, 17.0852, 16.499, 16.2686, 15.6844, 15.2234, 14.9732, 14.3356, 14.2286, 13.7262, 13.3284, 13.1048, 12.5962, 12.3562, 12.1272, 11.4184, 11.4974, 11.0822, 10.856, 10.48, 10.2834, 10.0208, 9.637, 9.51739999999999, 9.05759999999999, 8.74760000000001, 8.42700000000001, 8.1326, 8.2372, 8.2788, 7.6776, 7.79259999999999, 7.1952, 6.9564, 6.6454, 6.87, 6.5428, 6.19999999999999, 6.02940000000001, 5.62780000000001, 5.6782, 5.792, 5.35159999999999, 5.28319999999999, 5.0394, 5.07480000000001, 4.49119999999999, 4.84899999999999, 4.696, 4.54040000000001, 4.07300000000001, 4.37139999999999, 3.7216,
- 3.7328, 3.42080000000001, 3.41839999999999, 3.94239999999999, 3.27719999999999, 3.411, 3.13079999999999, 2.76900000000001, 2.92580000000001, 2.68279999999999, 2.75020000000001, 2.70599999999999, 2.3886, 3.01859999999999, 2.45179999999999, 2.92699999999999, 2.41720000000001, 2.41139999999999, 2.03299999999999, 2.51240000000001, 2.5564, 2.60079999999999, 2.41720000000001, 1.80439999999999, 1.99700000000001, 2.45480000000001, 1.8948, 2.2346, 2.30860000000001, 2.15479999999999, 1.88419999999999, 1.6508, 0.677199999999999, 1.72540000000001, 1.4752, 1.72280000000001, 1.66139999999999, 1.16759999999999, 1.79300000000001, 1.00059999999999, 0.905200000000008, 0.659999999999997, 1.55879999999999, 1.1636, 0.688199999999995, 0.712600000000009, 0.450199999999995, 1.1978, 0.975599999999986, 0.165400000000005, 1.727, 1.19739999999999, -0.252600000000001, 1.13460000000001, 1.3048, 1.19479999999999, 0.313400000000001, 0.878999999999991, 1.12039999999999, 0.853000000000009,
- 1.67920000000001, 0.856999999999999, 0.448599999999999, 1.2362, 0.953399999999988, 1.02859999999998, 0.563199999999995, 0.663000000000011, 0.723000000000013, 0.756599999999992, 0.256599999999992, -0.837600000000009, 0.620000000000005, 0.821599999999989, 0.216600000000028, 0.205600000000004, 0.220199999999977, 0.372599999999977, 0.334400000000016, 0.928400000000011, 0.972800000000007, 0.192400000000021, 0.487199999999973, -0.413000000000011, 0.807000000000016, 0.120600000000024, 0.769000000000005, 0.870799999999974, 0.66500000000002, 0.118200000000002, 0.401200000000017, 0.635199999999998, 0.135400000000004, 0.175599999999974, 1.16059999999999, 0.34620000000001, 0.521400000000028, -0.586599999999976, -1.16480000000001, 0.968399999999974, 0.836999999999989, 0.779600000000016, 0.985799999999983, },
- // precision 7
- { 91, 89.4934, 87.9758, 86.4574, 84.9718, 83.4954, 81.5302, 80.0756, 78.6374, 77.1782, 75.7888, 73.9522, 72.592, 71.2532, 69.9086, 68.5938, 66.9474, 65.6796, 64.4394, 63.2176, 61.9768, 60.4214, 59.2528, 58.0102, 56.8658, 55.7278, 54.3044, 53.1316, 52.093, 51.0032, 49.9092, 48.6306, 47.5294, 46.5756, 45.6508, 44.662, 43.552, 42.3724, 41.617, 40.5754, 39.7872, 38.8444, 37.7988, 36.8606, 36.2118, 35.3566, 34.4476, 33.5882, 32.6816, 32.0824, 31.0258, 30.6048, 29.4436, 28.7274, 27.957, 27.147, 26.4364, 25.7592, 25.3386, 24.781, 23.8028, 23.656, 22.6544, 21.996, 21.4718, 21.1544, 20.6098, 19.5956, 19.0616, 18.5758, 18.4878, 17.5244, 17.2146, 16.724, 15.8722, 15.5198, 15.0414, 14.941, 14.9048, 13.87, 13.4304, 13.028, 12.4708, 12.37, 12.0624, 11.4668, 11.5532, 11.4352, 11.2564, 10.2744, 10.2118, 9.74720000000002, 10.1456, 9.2928, 8.75040000000001, 8.55279999999999, 8.97899999999998, 8.21019999999999, 8.18340000000001, 7.3494, 7.32499999999999, 7.66140000000001, 6.90300000000002,
- 7.25439999999998, 6.9042, 7.21499999999997, 6.28640000000001, 6.08139999999997, 6.6764, 6.30099999999999, 5.13900000000001, 5.65800000000002, 5.17320000000001, 4.59019999999998, 4.9538, 5.08280000000002, 4.92200000000003, 4.99020000000002, 4.7328, 5.4538, 4.11360000000002, 4.22340000000003, 4.08780000000002, 3.70800000000003, 4.15559999999999, 4.18520000000001, 3.63720000000001, 3.68220000000002, 3.77960000000002, 3.6078, 2.49160000000001, 3.13099999999997, 2.5376, 3.19880000000001, 3.21100000000001, 2.4502, 3.52820000000003, 2.91199999999998, 3.04480000000001, 2.7432, 2.85239999999999, 2.79880000000003, 2.78579999999999, 1.88679999999999, 2.98860000000002, 2.50639999999999, 1.91239999999999, 2.66160000000002, 2.46820000000002, 1.58199999999999, 1.30399999999997, 2.27379999999999, 2.68939999999998, 1.32900000000001, 3.10599999999999, 1.69080000000002, 2.13740000000001, 2.53219999999999, 1.88479999999998, 1.33240000000001, 1.45119999999997, 1.17899999999997,
- 2.44119999999998, 1.60659999999996, 2.16700000000003, 0.77940000000001, 2.37900000000002, 2.06700000000001, 1.46000000000004, 2.91160000000002, 1.69200000000001, 0.954600000000028, 2.49300000000005, 2.2722, 1.33500000000004, 2.44899999999996, 1.20140000000004, 3.07380000000001, 2.09739999999999, 2.85640000000001, 2.29960000000005, 2.40899999999999, 1.97040000000004, 0.809799999999996, 1.65279999999996, 2.59979999999996, 0.95799999999997, 2.06799999999998, 2.32780000000002, 4.20159999999998, 1.96320000000003, 1.86400000000003, 1.42999999999995, 3.77940000000001, 1.27200000000005, 1.86440000000005, 2.20600000000002, 3.21900000000005, 1.5154, 2.61019999999996, },
- // precision 8
- { 183.2152, 180.2454, 177.2096, 173.6652, 170.6312, 167.6822, 164.249, 161.3296, 158.0038, 155.2074, 152.4612, 149.27, 146.5178, 143.4412, 140.8032, 138.1634, 135.1688, 132.6074, 129.6946, 127.2664, 124.8228, 122.0432, 119.6824, 116.9464, 114.6268, 112.2626, 109.8376, 107.4034, 104.8956, 102.8522, 100.7638, 98.3552, 96.3556, 93.7526, 91.9292, 89.8954, 87.8198, 85.7668, 83.298, 81.6688, 79.9466, 77.9746, 76.1672, 74.3474, 72.3028, 70.8912, 69.114, 67.4646, 65.9744, 64.4092, 62.6022, 60.843, 59.5684, 58.1652, 56.5426, 55.4152, 53.5388, 52.3592, 51.1366, 49.486, 48.3918, 46.5076, 45.509, 44.3834, 43.3498, 42.0668, 40.7346, 40.1228, 38.4528, 37.7, 36.644, 36.0518, 34.5774, 33.9068, 32.432, 32.1666, 30.434, 29.6644, 28.4894, 27.6312, 26.3804, 26.292, 25.5496000000001, 25.0234, 24.8206, 22.6146, 22.4188, 22.117, 20.6762, 20.6576, 19.7864, 19.509, 18.5334, 17.9204, 17.772, 16.2924, 16.8654, 15.1836, 15.745, 15.1316, 15.0386, 14.0136, 13.6342, 12.6196, 12.1866, 12.4281999999999,
- 11.3324, 10.4794000000001, 11.5038, 10.129, 9.52800000000002, 10.3203999999999, 9.46299999999997, 9.79280000000006, 9.12300000000005, 8.74180000000001, 9.2192, 7.51020000000005, 7.60659999999996, 7.01840000000004, 7.22239999999999, 7.40139999999997, 6.76179999999999, 7.14359999999999, 5.65060000000005, 5.63779999999997, 5.76599999999996, 6.75139999999999, 5.57759999999996, 3.73220000000003, 5.8048, 5.63019999999995, 4.93359999999996, 3.47979999999995, 4.33879999999999, 3.98940000000005, 3.81960000000004, 3.31359999999995, 3.23080000000004, 3.4588, 3.08159999999998, 3.4076, 3.00639999999999, 2.38779999999997, 2.61900000000003, 1.99800000000005, 3.34820000000002, 2.95060000000001, 0.990999999999985, 2.11440000000005, 2.20299999999997, 2.82219999999995, 2.73239999999998, 2.7826, 3.76660000000004, 2.26480000000004, 2.31280000000004, 2.40819999999997, 2.75360000000001, 3.33759999999995, 2.71559999999999, 1.7478000000001, 1.42920000000004, 2.39300000000003,
- 2.22779999999989, 2.34339999999997, 0.87259999999992, 3.88400000000001, 1.80600000000004, 1.91759999999999, 1.16779999999994, 1.50320000000011, 2.52500000000009, 0.226400000000012, 2.31500000000005, 0.930000000000064, 1.25199999999995, 2.14959999999996, 0.0407999999999902, 2.5447999999999, 1.32960000000003, 0.197400000000016, 2.52620000000002, 3.33279999999991, -1.34300000000007, 0.422199999999975, 0.917200000000093, 1.12920000000008, 1.46060000000011, 1.45779999999991, 2.8728000000001, 3.33359999999993, -1.34079999999994, 1.57680000000005, 0.363000000000056, 1.40740000000005, 0.656600000000026, 0.801400000000058, -0.454600000000028, 1.51919999999996, },
- // precision 9
- { 368, 361.8294, 355.2452, 348.6698, 342.1464, 336.2024, 329.8782, 323.6598, 317.462, 311.2826, 305.7102, 299.7416, 293.9366, 288.1046, 282.285, 277.0668, 271.306, 265.8448, 260.301, 254.9886, 250.2422, 244.8138, 239.7074, 234.7428, 229.8402, 225.1664, 220.3534, 215.594, 210.6886, 205.7876, 201.65, 197.228, 192.8036, 188.1666, 184.0818, 180.0824, 176.2574, 172.302, 168.1644, 164.0056, 160.3802, 156.7192, 152.5234, 149.2084, 145.831, 142.485, 139.1112, 135.4764, 131.76, 129.3368, 126.5538, 122.5058, 119.2646, 116.5902, 113.3818, 110.8998, 107.9532, 105.2062, 102.2798, 99.4728, 96.9582, 94.3292, 92.171, 89.7809999999999, 87.5716, 84.7048, 82.5322, 79.875, 78.3972, 75.3464, 73.7274, 71.2834, 70.1444, 68.4263999999999, 66.0166, 64.018, 62.0437999999999, 60.3399999999999, 58.6856, 57.9836, 55.0311999999999, 54.6769999999999, 52.3188, 51.4846, 49.4423999999999, 47.739, 46.1487999999999, 44.9202, 43.4059999999999, 42.5342000000001, 41.2834, 38.8954000000001, 38.3286000000001,
- 36.2146, 36.6684, 35.9946, 33.123, 33.4338, 31.7378000000001, 29.076, 28.9692, 27.4964, 27.0998, 25.9864, 26.7754, 24.3208, 23.4838, 22.7388000000001, 24.0758000000001, 21.9097999999999, 20.9728, 19.9228000000001, 19.9292, 16.617, 17.05, 18.2996000000001, 15.6128000000001, 15.7392, 14.5174, 13.6322, 12.2583999999999, 13.3766000000001, 11.423, 13.1232, 9.51639999999998, 10.5938000000001, 9.59719999999993, 8.12220000000002, 9.76739999999995, 7.50440000000003, 7.56999999999994, 6.70440000000008, 6.41419999999994, 6.71019999999999, 5.60940000000005, 4.65219999999999, 6.84099999999989, 3.4072000000001, 3.97859999999991, 3.32760000000007, 5.52160000000003, 3.31860000000006, 2.06940000000009, 4.35400000000004, 1.57500000000005, 0.280799999999999, 2.12879999999996, -0.214799999999968, -0.0378000000000611, -0.658200000000079, 0.654800000000023, -0.0697999999999865, 0.858400000000074, -2.52700000000004, -2.1751999999999, -3.35539999999992, -1.04019999999991,
- -0.651000000000067, -2.14439999999991, -1.96659999999997, -3.97939999999994, -0.604400000000169, -3.08260000000018, -3.39159999999993, -5.29640000000018, -5.38920000000007, -5.08759999999984, -4.69900000000007, -5.23720000000003, -3.15779999999995, -4.97879999999986, -4.89899999999989, -7.48880000000008, -5.94799999999987, -5.68060000000014, -6.67180000000008, -4.70499999999993, -7.27779999999984, -4.6579999999999, -4.4362000000001, -4.32139999999981, -5.18859999999995, -6.66879999999992, -6.48399999999992, -5.1260000000002, -4.4032000000002, -6.13500000000022, -5.80819999999994, -4.16719999999987, -4.15039999999999, -7.45600000000013, -7.24080000000004, -9.83179999999993, -5.80420000000004, -8.6561999999999, -6.99940000000015, -10.5473999999999, -7.34139999999979, -6.80999999999995, -6.29719999999998, -6.23199999999997, },
- // precision 10
- { 737.1256, 724.4234, 711.1064, 698.4732, 685.4636, 673.0644, 660.488, 647.9654, 636.0832, 623.7864, 612.1992, 600.2176, 588.5228, 577.1716, 565.7752, 554.899, 543.6126, 532.6492, 521.9474, 511.5214, 501.1064, 490.6364, 480.2468, 470.4588, 460.3832, 451.0584, 440.8606, 431.3868, 422.5062, 413.1862, 404.463, 395.339, 386.1936, 378.1292, 369.1854, 361.2908, 353.3324, 344.8518, 337.5204, 329.4854, 321.9318, 314.552, 306.4658, 299.4256, 292.849, 286.152, 278.8956, 271.8792, 265.118, 258.62, 252.5132, 245.9322, 239.7726, 233.6086, 227.5332, 222.5918, 216.4294, 210.7662, 205.4106, 199.7338, 194.9012, 188.4486, 183.1556, 178.6338, 173.7312, 169.6264, 163.9526, 159.8742, 155.8326, 151.1966, 147.5594, 143.07, 140.037, 134.1804, 131.071, 127.4884, 124.0848, 120.2944, 117.333, 112.9626, 110.2902, 107.0814, 103.0334, 99.4832000000001, 96.3899999999999, 93.7202000000002, 90.1714000000002, 87.2357999999999, 85.9346, 82.8910000000001, 80.0264000000002, 78.3834000000002,
- 75.1543999999999, 73.8683999999998, 70.9895999999999, 69.4367999999999, 64.8701999999998, 65.0408000000002, 61.6738, 59.5207999999998, 57.0158000000001, 54.2302, 53.0962, 50.4985999999999, 52.2588000000001, 47.3914, 45.6244000000002, 42.8377999999998, 43.0072, 40.6516000000001, 40.2453999999998, 35.2136, 36.4546, 33.7849999999999, 33.2294000000002, 32.4679999999998, 30.8670000000002, 28.6507999999999, 28.9099999999999, 27.5983999999999, 26.1619999999998, 24.5563999999999, 23.2328000000002, 21.9484000000002, 21.5902000000001, 21.3346000000001, 17.7031999999999, 20.6111999999998, 19.5545999999999, 15.7375999999999, 17.0720000000001, 16.9517999999998, 15.326, 13.1817999999998, 14.6925999999999, 13.0859999999998, 13.2754, 10.8697999999999, 11.248, 7.3768, 4.72339999999986, 7.97899999999981, 8.7503999999999, 7.68119999999999, 9.7199999999998, 7.73919999999998, 5.6224000000002, 7.44560000000001, 6.6601999999998, 5.9058, 4.00199999999995, 4.51699999999983,
- 4.68240000000014, 3.86220000000003, 5.13639999999987, 5.98500000000013, 2.47719999999981, 2.61999999999989, 1.62800000000016, 4.65000000000009, 0.225599999999758, 0.831000000000131, -0.359400000000278, 1.27599999999984, -2.92559999999958, -0.0303999999996449, 2.37079999999969, -2.0033999999996, 0.804600000000391, 0.30199999999968, 1.1247999999996, -2.6880000000001, 0.0321999999996478, -1.18099999999959, -3.9402, -1.47940000000017, -0.188400000000001, -2.10720000000038, -2.04159999999956, -3.12880000000041, -4.16160000000036, -0.612799999999879, -3.48719999999958, -8.17900000000009, -5.37780000000021, -4.01379999999972, -5.58259999999973, -5.73719999999958, -7.66799999999967, -5.69520000000011, -1.1247999999996, -5.58520000000044, -8.04560000000038, -4.64840000000004, -11.6468000000004, -7.97519999999986, -5.78300000000036, -7.67420000000038, -10.6328000000003, -9.81720000000041, },
- // precision 11
- { 1476, 1449.6014, 1423.5802, 1397.7942, 1372.3042, 1347.2062, 1321.8402, 1297.2292, 1272.9462, 1248.9926, 1225.3026, 1201.4252, 1178.0578, 1155.6092, 1132.626, 1110.5568, 1088.527, 1066.5154, 1045.1874, 1024.3878, 1003.37, 982.1972, 962.5728, 942.1012, 922.9668, 903.292, 884.0772, 864.8578, 846.6562, 828.041, 809.714, 792.3112, 775.1806, 757.9854, 740.656, 724.346, 707.5154, 691.8378, 675.7448, 659.6722, 645.5722, 630.1462, 614.4124, 600.8728, 585.898, 572.408, 558.4926, 544.4938, 531.6776, 517.282, 505.7704, 493.1012, 480.7388, 467.6876, 456.1872, 445.5048, 433.0214, 420.806, 411.409, 400.4144, 389.4294, 379.2286, 369.651, 360.6156, 350.337, 342.083, 332.1538, 322.5094, 315.01, 305.6686, 298.1678, 287.8116, 280.9978, 271.9204, 265.3286, 257.5706, 249.6014, 242.544, 235.5976, 229.583, 220.9438, 214.672, 208.2786, 201.8628, 195.1834, 191.505, 186.1816, 178.5188, 172.2294, 167.8908, 161.0194, 158.052, 151.4588, 148.1596, 143.4344, 138.5238, 133.13, 127.6374, 124.8162,
- 118.7894, 117.3984, 114.6078, 109.0858, 105.1036, 103.6258, 98.6018000000004, 95.7618000000002, 93.5821999999998, 88.5900000000001, 86.9992000000002, 82.8800000000001, 80.4539999999997, 74.6981999999998, 74.3644000000004, 73.2914000000001, 65.5709999999999, 66.9232000000002, 65.1913999999997, 62.5882000000001, 61.5702000000001, 55.7035999999998, 56.1764000000003, 52.7596000000003, 53.0302000000001, 49.0609999999997, 48.4694, 44.933, 46.0474000000004, 44.7165999999997, 41.9416000000001, 39.9207999999999, 35.6328000000003, 35.5276000000003, 33.1934000000001, 33.2371999999996, 33.3864000000003, 33.9228000000003, 30.2371999999996, 29.1373999999996, 25.2272000000003, 24.2942000000003, 19.8338000000003, 18.9005999999999, 23.0907999999999, 21.8544000000002, 19.5176000000001, 15.4147999999996, 16.9314000000004, 18.6737999999996, 12.9877999999999, 14.3688000000002, 12.0447999999997, 15.5219999999999, 12.5299999999997, 14.5940000000001, 14.3131999999996, 9.45499999999993,
- 12.9441999999999, 3.91139999999996, 13.1373999999996, 5.44720000000052, 9.82779999999912, 7.87279999999919, 3.67760000000089, 5.46980000000076, 5.55099999999948, 5.65979999999945, 3.89439999999922, 3.1275999999998, 5.65140000000065, 6.3062000000009, 3.90799999999945, 1.87060000000019, 5.17020000000048, 2.46680000000015, 0.770000000000437, -3.72340000000077, 1.16400000000067, 8.05340000000069, 0.135399999999208, 2.15940000000046, 0.766999999999825, 1.0594000000001, 3.15500000000065, -0.287399999999252, 2.37219999999979, -2.86620000000039, -1.63199999999961, -2.22979999999916, -0.15519999999924, -1.46039999999994, -0.262199999999211, -2.34460000000036, -2.8078000000005, -3.22179999999935, -5.60159999999996, -8.42200000000048, -9.43740000000071, 0.161799999999857, -10.4755999999998, -10.0823999999993, },
- // precision 12
- { 2953, 2900.4782, 2848.3568, 2796.3666, 2745.324, 2694.9598, 2644.648, 2595.539, 2546.1474, 2498.2576, 2450.8376, 2403.6076, 2357.451, 2311.38, 2266.4104, 2221.5638, 2176.9676, 2134.193, 2090.838, 2048.8548, 2007.018, 1966.1742, 1925.4482, 1885.1294, 1846.4776, 1807.4044, 1768.8724, 1731.3732, 1693.4304, 1657.5326, 1621.949, 1586.5532, 1551.7256, 1517.6182, 1483.5186, 1450.4528, 1417.865, 1385.7164, 1352.6828, 1322.6708, 1291.8312, 1260.9036, 1231.476, 1201.8652, 1173.6718, 1145.757, 1119.2072, 1092.2828, 1065.0434, 1038.6264, 1014.3192, 988.5746, 965.0816, 940.1176, 917.9796, 894.5576, 871.1858, 849.9144, 827.1142, 805.0818, 783.9664, 763.9096, 742.0816, 724.3962, 706.3454, 688.018, 667.4214, 650.3106, 633.0686, 613.8094, 597.818, 581.4248, 563.834, 547.363, 531.5066, 520.455400000001, 505.583199999999, 488.366, 476.480799999999, 459.7682, 450.0522, 434.328799999999, 423.952799999999, 408.727000000001, 399.079400000001, 387.252200000001, 373.987999999999,
- 360.852000000001, 351.6394, 339.642, 330.902400000001, 322.661599999999, 311.662200000001, 301.3254, 291.7484, 279.939200000001, 276.7508, 263.215200000001, 254.811400000001, 245.5494, 242.306399999999, 234.8734, 223.787200000001, 217.7156, 212.0196, 200.793, 195.9748, 189.0702, 182.449199999999, 177.2772, 170.2336, 164.741, 158.613600000001, 155.311, 147.5964, 142.837, 137.3724, 132.0162, 130.0424, 121.9804, 120.451800000001, 114.8968, 111.585999999999, 105.933199999999, 101.705, 98.5141999999996, 95.0488000000005, 89.7880000000005, 91.4750000000004, 83.7764000000006, 80.9698000000008, 72.8574000000008, 73.1615999999995, 67.5838000000003, 62.6263999999992, 63.2638000000006, 66.0977999999996, 52.0843999999997, 58.9956000000002, 47.0912000000008, 46.4956000000002, 48.4383999999991, 47.1082000000006, 43.2392, 37.2759999999998, 40.0283999999992, 35.1864000000005, 35.8595999999998, 32.0998, 28.027, 23.6694000000007, 33.8266000000003, 26.3736000000008,
- 27.2008000000005, 21.3245999999999, 26.4115999999995, 23.4521999999997, 19.5013999999992, 19.8513999999996, 10.7492000000002, 18.6424000000006, 13.1265999999996, 18.2436000000016, 6.71860000000015, 3.39459999999963, 6.33759999999893, 7.76719999999841, 0.813999999998487, 3.82819999999992, 0.826199999999517, 8.07440000000133, -1.59080000000176, 5.01780000000144, 0.455399999998917, -0.24199999999837, 0.174800000000687, -9.07640000000174, -4.20160000000033, -3.77520000000004, -4.75179999999818, -5.3724000000002, -8.90680000000066, -6.10239999999976, -5.74120000000039, -9.95339999999851, -3.86339999999836, -13.7304000000004, -16.2710000000006, -7.51359999999841, -3.30679999999847, -13.1339999999982, -10.0551999999989, -6.72019999999975, -8.59660000000076, -10.9307999999983, -1.8775999999998, -4.82259999999951, -13.7788, -21.6470000000008, -10.6735999999983, -15.7799999999988, },
- // precision 13
- { 5907.5052, 5802.2672, 5697.347, 5593.5794, 5491.2622, 5390.5514, 5290.3376, 5191.6952, 5093.5988, 4997.3552, 4902.5972, 4808.3082, 4715.5646, 4624.109, 4533.8216, 4444.4344, 4356.3802, 4269.2962, 4183.3784, 4098.292, 4014.79, 3932.4574, 3850.6036, 3771.2712, 3691.7708, 3615.099, 3538.1858, 3463.4746, 3388.8496, 3315.6794, 3244.5448, 3173.7516, 3103.3106, 3033.6094, 2966.5642, 2900.794, 2833.7256, 2769.81, 2707.3196, 2644.0778, 2583.9916, 2523.4662, 2464.124, 2406.073, 2347.0362, 2292.1006, 2238.1716, 2182.7514, 2128.4884, 2077.1314, 2025.037, 1975.3756, 1928.933, 1879.311, 1831.0006, 1783.2144, 1738.3096, 1694.5144, 1649.024, 1606.847, 1564.7528, 1525.3168, 1482.5372, 1443.9668, 1406.5074, 1365.867, 1329.2186, 1295.4186, 1257.9716, 1225.339, 1193.2972, 1156.3578, 1125.8686, 1091.187, 1061.4094, 1029.4188, 1000.9126, 972.3272, 944.004199999999, 915.7592, 889.965, 862.834200000001, 840.4254, 812.598399999999, 785.924200000001, 763.050999999999, 741.793799999999, 721.466,
- 699.040799999999, 677.997200000002, 649.866999999998, 634.911800000002, 609.8694, 591.981599999999, 570.2922, 557.129199999999, 538.3858, 521.872599999999, 502.951400000002, 495.776399999999, 475.171399999999, 459.751, 439.995200000001, 426.708999999999, 413.7016, 402.3868, 387.262599999998, 372.0524, 357.050999999999, 342.5098, 334.849200000001, 322.529399999999, 311.613799999999, 295.848000000002, 289.273000000001, 274.093000000001, 263.329600000001, 251.389599999999, 245.7392, 231.9614, 229.7952, 217.155200000001, 208.9588, 199.016599999999, 190.839199999999, 180.6976, 176.272799999999, 166.976999999999, 162.5252, 151.196400000001, 149.386999999999, 133.981199999998, 130.0586, 130.164000000001, 122.053400000001, 110.7428, 108.1276, 106.232400000001, 100.381600000001, 98.7668000000012, 86.6440000000002, 79.9768000000004, 82.4722000000002, 68.7026000000005, 70.1186000000016, 71.9948000000004, 58.998599999999, 59.0492000000013, 56.9818000000014, 47.5338000000011,
- 42.9928, 51.1591999999982, 37.2740000000013, 42.7220000000016, 31.3734000000004, 26.8090000000011, 25.8934000000008, 26.5286000000015, 29.5442000000003, 19.3503999999994, 26.0760000000009, 17.9527999999991, 14.8419999999969, 10.4683999999979, 8.65899999999965, 9.86720000000059, 4.34139999999752, -0.907800000000861, -3.32080000000133, -0.936199999996461, -11.9916000000012, -8.87000000000262, -6.33099999999831, -11.3366000000024, -15.9207999999999, -9.34659999999712, -15.5034000000014, -19.2097999999969, -15.357799999998, -28.2235999999975, -30.6898000000001, -19.3271999999997, -25.6083999999973, -24.409599999999, -13.6385999999984, -33.4473999999973, -32.6949999999997, -28.9063999999998, -31.7483999999968, -32.2935999999972, -35.8329999999987, -47.620600000002, -39.0855999999985, -33.1434000000008, -46.1371999999974, -37.5892000000022, -46.8164000000033, -47.3142000000007, -60.2914000000019, -37.7575999999972, },
- // precision 14
- { 11816.475, 11605.0046, 11395.3792, 11188.7504, 10984.1814, 10782.0086, 10582.0072, 10384.503, 10189.178, 9996.2738, 9806.0344, 9617.9798, 9431.394, 9248.7784, 9067.6894, 8889.6824, 8712.9134, 8538.8624, 8368.4944, 8197.7956, 8031.8916, 7866.6316, 7703.733, 7544.5726, 7386.204, 7230.666, 7077.8516, 6926.7886, 6778.6902, 6631.9632, 6487.304, 6346.7486, 6206.4408, 6070.202, 5935.2576, 5799.924, 5671.0324, 5541.9788, 5414.6112, 5290.0274, 5166.723, 5047.6906, 4929.162, 4815.1406, 4699.127, 4588.5606, 4477.7394, 4369.4014, 4264.2728, 4155.9224, 4055.581, 3955.505, 3856.9618, 3761.3828, 3666.9702, 3575.7764, 3482.4132, 3395.0186, 3305.8852, 3221.415, 3138.6024, 3056.296, 2970.4494, 2896.1526, 2816.8008, 2740.2156, 2670.497, 2594.1458, 2527.111, 2460.8168, 2387.5114, 2322.9498, 2260.6752, 2194.2686, 2133.7792, 2074.767, 2015.204, 1959.4226, 1898.6502, 1850.006, 1792.849, 1741.4838, 1687.9778, 1638.1322, 1589.3266, 1543.1394, 1496.8266, 1447.8516, 1402.7354, 1361.9606,
- 1327.0692, 1285.4106, 1241.8112, 1201.6726, 1161.973, 1130.261, 1094.2036, 1048.2036, 1020.6436, 990.901400000002, 961.199800000002, 924.769800000002, 899.526400000002, 872.346400000002, 834.375, 810.432000000001, 780.659800000001, 756.013800000001, 733.479399999997, 707.923999999999, 673.858, 652.222399999999, 636.572399999997, 615.738599999997, 586.696400000001, 564.147199999999, 541.679600000003, 523.943599999999, 505.714599999999, 475.729599999999, 461.779600000002, 449.750800000002, 439.020799999998, 412.7886, 400.245600000002, 383.188199999997, 362.079599999997, 357.533799999997, 334.319000000003, 327.553399999997, 308.559399999998, 291.270199999999, 279.351999999999, 271.791400000002, 252.576999999997, 247.482400000001, 236.174800000001, 218.774599999997, 220.155200000001, 208.794399999999, 201.223599999998, 182.995600000002, 185.5268, 164.547400000003, 176.5962, 150.689599999998, 157.8004, 138.378799999999, 134.021200000003, 117.614399999999,
- 108.194000000003, 97.0696000000025, 89.6042000000016, 95.6030000000028, 84.7810000000027, 72.635000000002, 77.3482000000004, 59.4907999999996, 55.5875999999989, 50.7346000000034, 61.3916000000027, 50.9149999999936, 39.0384000000049, 58.9395999999979, 29.633600000001, 28.2032000000036, 26.0078000000067, 17.0387999999948, 9.22000000000116, 13.8387999999977, 8.07240000000456, 14.1549999999988, 15.3570000000036, 3.42660000000615, 6.24820000000182, -2.96940000000177, -8.79940000000352, -5.97860000000219, -14.4048000000039, -3.4143999999942, -13.0148000000045, -11.6977999999945, -25.7878000000055, -22.3185999999987, -24.409599999999, -31.9756000000052, -18.9722000000038, -22.8678000000073, -30.8972000000067, -32.3715999999986, -22.3907999999938, -43.6720000000059, -35.9038, -39.7492000000057, -54.1641999999993, -45.2749999999942, -42.2989999999991, -44.1089999999967, -64.3564000000042, -49.9551999999967, -42.6116000000038, },
- // precision 15
- { 23634.0036, 23210.8034, 22792.4744, 22379.1524, 21969.7928, 21565.326, 21165.3532, 20770.2806, 20379.9892, 19994.7098, 19613.318, 19236.799, 18865.4382, 18498.8244, 18136.5138, 17778.8668, 17426.2344, 17079.32, 16734.778, 16397.2418, 16063.3324, 15734.0232, 15409.731, 15088.728, 14772.9896, 14464.1402, 14157.5588, 13855.5958, 13559.3296, 13264.9096, 12978.326, 12692.0826, 12413.8816, 12137.3192, 11870.2326, 11602.5554, 11340.3142, 11079.613, 10829.5908, 10583.5466, 10334.0344, 10095.5072, 9859.694, 9625.2822, 9395.7862, 9174.0586, 8957.3164, 8738.064, 8524.155, 8313.7396, 8116.9168, 7913.542, 7718.4778, 7521.65, 7335.5596, 7154.2906, 6968.7396, 6786.3996, 6613.236, 6437.406, 6270.6598, 6107.7958, 5945.7174, 5787.6784, 5635.5784, 5482.308, 5337.9784, 5190.0864, 5045.9158, 4919.1386, 4771.817, 4645.7742, 4518.4774, 4385.5454, 4262.6622, 4142.74679999999, 4015.5318, 3897.9276, 3790.7764, 3685.13800000001, 3573.6274, 3467.9706, 3368.61079999999, 3271.5202, 3170.3848,
- 3076.4656, 2982.38400000001, 2888.4664, 2806.4868, 2711.9564, 2634.1434, 2551.3204, 2469.7662, 2396.61139999999, 2318.9902, 2243.8658, 2171.9246, 2105.01360000001, 2028.8536, 1960.9952, 1901.4096, 1841.86079999999, 1777.54700000001, 1714.5802, 1654.65059999999, 1596.311, 1546.2016, 1492.3296, 1433.8974, 1383.84600000001, 1339.4152, 1293.5518, 1245.8686, 1193.50659999999, 1162.27959999999, 1107.19439999999, 1069.18060000001, 1035.09179999999, 999.679000000004, 957.679999999993, 925.300199999998, 888.099400000006, 848.638600000006, 818.156400000007, 796.748399999997, 752.139200000005, 725.271200000003, 692.216, 671.633600000001, 647.939799999993, 621.670599999998, 575.398799999995, 561.226599999995, 532.237999999998, 521.787599999996, 483.095799999996, 467.049599999998, 465.286399999997, 415.548599999995, 401.047399999996, 380.607999999993, 377.362599999993, 347.258799999996, 338.371599999999, 310.096999999994, 301.409199999995, 276.280799999993, 265.586800000005,
- 258.994399999996, 223.915999999997, 215.925399999993, 213.503800000006, 191.045400000003, 166.718200000003, 166.259000000005, 162.941200000001, 148.829400000002, 141.645999999993, 123.535399999993, 122.329800000007, 89.473399999988, 80.1962000000058, 77.5457999999926, 59.1056000000099, 83.3509999999951, 52.2906000000075, 36.3979999999865, 40.6558000000077, 42.0003999999899, 19.6630000000005, 19.7153999999864, -8.38539999999921, -0.692799999989802, 0.854800000000978, 3.23219999999856, -3.89040000000386, -5.25880000001052, -24.9052000000083, -22.6837999999989, -26.4286000000138, -34.997000000003, -37.0216000000073, -43.430400000012, -58.2390000000014, -68.8034000000043, -56.9245999999985, -57.8583999999973, -77.3097999999882, -73.2793999999994, -81.0738000000129, -87.4530000000086, -65.0254000000132, -57.296399999992, -96.2746000000043, -103.25, -96.081600000005, -91.5542000000132, -102.465200000006, -107.688599999994, -101.458000000013, -109.715800000005, },
- // precision 16
- { 47270, 46423.3584, 45585.7074, 44757.152, 43938.8416, 43130.9514, 42330.03, 41540.407, 40759.6348, 39988.206, 39226.5144, 38473.2096, 37729.795, 36997.268, 36272.6448, 35558.665, 34853.0248, 34157.4472, 33470.5204, 32793.5742, 32127.0194, 31469.4182, 30817.6136, 30178.6968, 29546.8908, 28922.8544, 28312.271, 27707.0924, 27114.0326, 26526.692, 25948.6336, 25383.7826, 24823.5998, 24272.2974, 23732.2572, 23201.4976, 22674.2796, 22163.6336, 21656.515, 21161.7362, 20669.9368, 20189.4424, 19717.3358, 19256.3744, 18795.9638, 18352.197, 17908.5738, 17474.391, 17052.918, 16637.2236, 16228.4602, 15823.3474, 15428.6974, 15043.0284, 14667.6278, 14297.4588, 13935.2882, 13578.5402, 13234.6032, 12882.1578, 12548.0728, 12219.231, 11898.0072, 11587.2626, 11279.9072, 10973.5048, 10678.5186, 10392.4876, 10105.2556, 9825.766, 9562.5444, 9294.2222, 9038.2352, 8784.848, 8533.2644, 8301.7776, 8058.30859999999, 7822.94579999999, 7599.11319999999, 7366.90779999999, 7161.217, 6957.53080000001,
- 6736.212, 6548.21220000001, 6343.06839999999, 6156.28719999999, 5975.15419999999, 5791.75719999999, 5621.32019999999, 5451.66, 5287.61040000001, 5118.09479999999, 4957.288, 4798.4246, 4662.17559999999, 4512.05900000001, 4364.68539999999, 4220.77720000001, 4082.67259999999, 3957.19519999999, 3842.15779999999, 3699.3328, 3583.01180000001, 3473.8964, 3338.66639999999, 3233.55559999999, 3117.799, 3008.111, 2909.69140000001, 2814.86499999999, 2719.46119999999, 2624.742, 2532.46979999999, 2444.7886, 2370.1868, 2272.45259999999, 2196.19260000001, 2117.90419999999, 2023.2972, 1969.76819999999, 1885.58979999999, 1833.2824, 1733.91200000001, 1682.54920000001, 1604.57980000001, 1556.11240000001, 1491.3064, 1421.71960000001, 1371.22899999999, 1322.1324, 1264.7892, 1196.23920000001, 1143.8474, 1088.67240000001, 1073.60380000001, 1023.11660000001, 959.036400000012, 927.433199999999, 906.792799999996, 853.433599999989, 841.873800000001, 791.1054, 756.899999999994,
- 704.343200000003, 672.495599999995, 622.790399999998, 611.254799999995, 567.283200000005, 519.406599999988, 519.188400000014, 495.312800000014, 451.350799999986, 443.973399999988, 431.882199999993, 392.027000000002, 380.924200000009, 345.128999999986, 298.901400000002, 287.771999999997, 272.625, 247.253000000026, 222.490600000019, 223.590000000026, 196.407599999977, 176.425999999978, 134.725199999986, 132.4804, 110.445599999977, 86.7939999999944, 56.7038000000175, 64.915399999998, 38.3726000000024, 37.1606000000029, 46.170999999973, 49.1716000000015, 15.3362000000197, 6.71639999997569, -34.8185999999987, -39.4476000000141, 12.6830000000191, -12.3331999999937, -50.6565999999875, -59.9538000000175, -65.1054000000004, -70.7576000000117, -106.325200000021, -126.852200000023, -110.227599999984, -132.885999999999, -113.897200000007, -142.713800000027, -151.145399999979, -150.799200000009, -177.756200000003, -156.036399999983, -182.735199999996, -177.259399999981,
- -198.663600000029, -174.577600000019, -193.84580000001, },
- // precision 17
- { 94541, 92848.811, 91174.019, 89517.558, 87879.9705, 86262.7565, 84663.5125, 83083.7435, 81521.7865, 79977.272, 78455.9465, 76950.219, 75465.432, 73994.152, 72546.71, 71115.2345, 69705.6765, 68314.937, 66944.2705, 65591.255, 64252.9485, 62938.016, 61636.8225, 60355.592, 59092.789, 57850.568, 56624.518, 55417.343, 54231.1415, 53067.387, 51903.526, 50774.649, 49657.6415, 48561.05, 47475.7575, 46410.159, 45364.852, 44327.053, 43318.4005, 42325.6165, 41348.4595, 40383.6265, 39436.77, 38509.502, 37594.035, 36695.939, 35818.6895, 34955.691, 34115.8095, 33293.949, 32465.0775, 31657.6715, 30877.2585, 30093.78, 29351.3695, 28594.1365, 27872.115, 27168.7465, 26477.076, 25774.541, 25106.5375, 24452.5135, 23815.5125, 23174.0655, 22555.2685, 21960.2065, 21376.3555, 20785.1925, 20211.517, 19657.0725, 19141.6865, 18579.737, 18081.3955, 17578.995, 17073.44, 16608.335, 16119.911, 15651.266, 15194.583, 14749.0495, 14343.4835, 13925.639, 13504.509, 13099.3885, 12691.2855, 12328.018,
- 11969.0345, 11596.5145, 11245.6355, 10917.6575, 10580.9785, 10277.8605, 9926.58100000001, 9605.538, 9300.42950000003, 8989.97850000003, 8728.73249999998, 8448.3235, 8175.31050000002, 7898.98700000002, 7629.79100000003, 7413.76199999999, 7149.92300000001, 6921.12650000001, 6677.1545, 6443.28000000003, 6278.23450000002, 6014.20049999998, 5791.20299999998, 5605.78450000001, 5438.48800000001, 5234.2255, 5059.6825, 4887.43349999998, 4682.935, 4496.31099999999, 4322.52250000002, 4191.42499999999, 4021.24200000003, 3900.64799999999, 3762.84250000003, 3609.98050000001, 3502.29599999997, 3363.84250000003, 3206.54849999998, 3079.70000000001, 2971.42300000001, 2867.80349999998, 2727.08100000001, 2630.74900000001, 2496.6165, 2440.902, 2356.19150000002, 2235.58199999999, 2120.54149999999, 2012.25449999998, 1933.35600000003, 1820.93099999998, 1761.54800000001, 1663.09350000002, 1578.84600000002, 1509.48149999999, 1427.3345, 1379.56150000001, 1306.68099999998, 1212.63449999999,
- 1084.17300000001, 1124.16450000001, 1060.69949999999, 1007.48849999998, 941.194499999983, 879.880500000028, 836.007500000007, 782.802000000025, 748.385499999975, 647.991500000004, 626.730500000005, 570.776000000013, 484.000500000024, 513.98550000001, 418.985499999952, 386.996999999974, 370.026500000036, 355.496999999974, 356.731499999994, 255.92200000002, 259.094000000041, 205.434499999974, 165.374500000034, 197.347500000033, 95.718499999959, 67.6165000000037, 54.6970000000438, 31.7395000000251, -15.8784999999916, 8.42500000004657, -26.3754999999655, -118.425500000012, -66.6629999999423, -42.9745000000112, -107.364999999991, -189.839000000036, -162.611499999999, -164.964999999967, -189.079999999958, -223.931499999948, -235.329999999958, -269.639500000048, -249.087999999989, -206.475499999942, -283.04449999996, -290.667000000016, -304.561499999953, -336.784499999951, -380.386500000022, -283.280499999993, -364.533000000054, -389.059499999974, -364.454000000027,
- -415.748000000021, -417.155000000028, },
- // precision 18
- { 189083, 185696.913, 182348.774, 179035.946, 175762.762, 172526.444, 169329.754, 166166.099, 163043.269, 159958.91, 156907.912, 153906.845, 150924.199, 147996.568, 145093.457, 142239.233, 139421.475, 136632.27, 133889.588, 131174.2, 128511.619, 125868.621, 123265.385, 120721.061, 118181.769, 115709.456, 113252.446, 110840.198, 108465.099, 106126.164, 103823.469, 101556.618, 99308.004, 97124.508, 94937.803, 92833.731, 90745.061, 88677.627, 86617.47, 84650.442, 82697.833, 80769.132, 78879.629, 77014.432, 75215.626, 73384.587, 71652.482, 69895.93, 68209.301, 66553.669, 64921.981, 63310.323, 61742.115, 60205.018, 58698.658, 57190.657, 55760.865, 54331.169, 52908.167, 51550.273, 50225.254, 48922.421, 47614.533, 46362.049, 45098.569, 43926.083, 42736.03, 41593.473, 40425.26, 39316.237, 38243.651, 37170.617, 36114.609, 35084.19, 34117.233, 33206.509, 32231.505, 31318.728, 30403.404, 29540.0550000001, 28679.236, 27825.862, 26965.216, 26179.148, 25462.08, 24645.952, 23922.523,
- 23198.144, 22529.128, 21762.4179999999, 21134.779, 20459.117, 19840.818, 19187.04, 18636.3689999999, 17982.831, 17439.7389999999, 16874.547, 16358.2169999999, 15835.684, 15352.914, 14823.681, 14329.313, 13816.897, 13342.874, 12880.882, 12491.648, 12021.254, 11625.392, 11293.7610000001, 10813.697, 10456.209, 10099.074, 9755.39000000001, 9393.18500000006, 9047.57900000003, 8657.98499999999, 8395.85900000005, 8033, 7736.95900000003, 7430.59699999995, 7258.47699999996, 6924.58200000005, 6691.29399999999, 6357.92500000005, 6202.05700000003, 5921.19700000004, 5628.28399999999, 5404.96799999999, 5226.71100000001, 4990.75600000005, 4799.77399999998, 4622.93099999998, 4472.478, 4171.78700000001, 3957.46299999999, 3868.95200000005, 3691.14300000004, 3474.63100000005, 3341.67200000002, 3109.14000000001, 3071.97400000005, 2796.40399999998, 2756.17799999996, 2611.46999999997, 2471.93000000005, 2382.26399999997, 2209.22400000005, 2142.28399999999, 2013.96100000001,
- 1911.18999999994, 1818.27099999995, 1668.47900000005, 1519.65800000005, 1469.67599999998, 1367.13800000004, 1248.52899999998, 1181.23600000003, 1022.71900000004, 1088.20700000005, 959.03600000008, 876.095999999903, 791.183999999892, 703.337000000058, 731.949999999953, 586.86400000006, 526.024999999907, 323.004999999888, 320.448000000091, 340.672999999952, 309.638999999966, 216.601999999955, 102.922999999952, 19.2399999999907, -0.114000000059605, -32.6240000000689, -89.3179999999702, -153.497999999905, -64.2970000000205, -143.695999999996, -259.497999999905, -253.017999999924, -213.948000000091, -397.590000000084, -434.006000000052, -403.475000000093, -297.958000000101, -404.317000000039, -528.898999999976, -506.621000000043, -513.205000000075, -479.351000000024, -596.139999999898, -527.016999999993, -664.681000000099, -680.306000000099, -704.050000000047, -850.486000000034, -757.43200000003, -713.308999999892, } };
-
- public static boolean isLinearCounting(int precision, double estimate) {
- return estimate <= thresholdData[precision - 4];
- }
-
- public static boolean isBiasCorrection(int numberOfBuckets, double estimate) {
- return estimate <= 5 * numberOfBuckets;
- }
-
- public static double biasCorrection(int precision, double estimate) {
- double[] estimateVector = rawEstimateData[precision - 4];
- SortedMap<Double, Integer> estimateDistances = calcDistances(estimate, estimateVector);
- int[] nearestNeighbors = getNearestNeighbors(estimateDistances);
- double bias = getBias(precision, nearestNeighbors);
- return estimate - bias;
- }
-
- public static SortedMap<Double, Integer> calcDistances(double estimate, double[] estimateVector) {
- SortedMap<Double, Integer> distances = new TreeMap<Double, Integer>();
- int index = 0;
- for (double anEstimateVector : estimateVector) {
- distances.put(Math.pow(estimate - anEstimateVector, 2), index++);
- }
- return distances;
- }
-
- public static double getBias(int precision, int[] nearestNeighbors) {
- double[] biasVector = biasData[precision - 4];
- double biasTotal = 0.0d;
- for (int nearestNeighbor : nearestNeighbors) {
- biasTotal += biasVector[nearestNeighbor];
- }
- return biasTotal / (nearestNeighbors.length);
- }
-
- public static int[] getNearestNeighbors(SortedMap<Double, Integer> distanceMap) {
- int[] nearest = new int[6];
- int i = 0;
- for (Integer index : distanceMap.values()) {
- nearest[i++] = index;
- if (i >= 6) {
- break;
- }
- }
- return nearest;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/JobLock.java b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
deleted file mode 100644
index 650a3f8..0000000
--- a/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.lock;
-
-/**
- */
-public interface JobLock {
- boolean lock();
-
- void unlock();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
deleted file mode 100644
index 52b34a2..0000000
--- a/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.lock;
-
-/**
- */
-public class MockJobLock implements JobLock {
- @Override
- public boolean lock() {
- return true;
- }
-
- @Override
- public void unlock() {
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
deleted file mode 100644
index 2bed5c6..0000000
--- a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.lock;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.imps.CuratorFrameworkState;
-import org.apache.curator.framework.recipes.locks.InterProcessMutex;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class ZookeeperJobLock implements JobLock {
- private Logger logger = LoggerFactory.getLogger(ZookeeperJobLock.class);
-
- private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
-
- private String scheduleID;
- private InterProcessMutex sharedLock;
- private CuratorFramework zkClient;
-
- @Override
- public boolean lock() {
- this.scheduleID = schedulerId();
- String ZKConnectString = getZKConnectString();
- if (StringUtils.isEmpty(ZKConnectString)) {
- throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
- }
-
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
- this.zkClient.start();
- this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
- boolean hasLock = false;
- try {
- hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
- } catch (Exception e) {
- logger.warn("error acquire lock", e);
- }
- if (!hasLock) {
- logger.warn("fail to acquire lock, scheduler has not been started");
- zkClient.close();
- return false;
- }
- return true;
- }
-
- @Override
- public void unlock() {
- releaseLock();
- }
-
- private String getZKConnectString() {
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- }
-
- private void releaseLock() {
- try {
- if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
- // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
- if (zkClient.checkExists().forPath(scheduleID) != null) {
- zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(scheduleID);
- }
- }
- } catch (Exception e) {
- logger.error("error release lock:" + scheduleID);
- throw new RuntimeException(e);
- }
- }
-
- private String schedulerId() {
- return ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java b/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
deleted file mode 100644
index 5508583..0000000
--- a/common/src/main/java/org/apache/kylin/common/mr/KylinMapper.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.mr;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.kylin.common.util.HadoopUtil;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/19/15.
- */
-public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
- protected void publishConfiguration(Configuration conf) {
- HadoopUtil.setCurrentConfiguration(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java b/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
deleted file mode 100644
index df28807..0000000
--- a/common/src/main/java/org/apache/kylin/common/mr/KylinReducer.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.mr;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.kylin.common.util.HadoopUtil;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/19/15.
- */
-public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
- protected void publishConfiguration(Configuration conf) {
- HadoopUtil.setCurrentConfiguration(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java b/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java
deleted file mode 100644
index c443b71..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/AclEntity.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-/**
- * @author xduo
- *
- */
-public interface AclEntity {
-
- public String getId();
-
-}
[12/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cmd/ShellCmdOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/ShellCmdOutput.java b/job/src/main/java/org/apache/kylin/job/cmd/ShellCmdOutput.java
deleted file mode 100644
index 2813596..0000000
--- a/job/src/main/java/org/apache/kylin/job/cmd/ShellCmdOutput.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cmd;
-
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xjiang
- *
- */
-public class ShellCmdOutput extends BaseCommandOutput implements ICommandOutput {
-
- protected static final Logger log = LoggerFactory.getLogger(ShellCmdOutput.class);
-
- protected StringBuilder output;
- protected int exitCode;
- protected JobStepStatusEnum status;
-
- public ShellCmdOutput() {
- init();
- }
-
- private void init() {
- output = new StringBuilder();
- exitCode = -1;
- status = JobStepStatusEnum.NEW;
- }
-
- @Override
- public JobStepStatusEnum getStatus() {
- return status;
- }
-
- @Override
- public void setStatus(JobStepStatusEnum s) {
- this.status = s;
- }
-
- @Override
- public String getOutput() {
- return output.toString();
- }
-
- @Override
- public void appendOutput(String message) {
- output.append(message).append(System.getProperty("line.separator"));
- log.debug(message);
- }
-
- @Override
- public int getExitCode() {
- return exitCode;
- }
-
- @Override
- public void setExitCode(int code) {
- exitCode = code;
- }
-
- @Override
- public void reset() {
- init();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/common/HadoopCmdOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/HadoopCmdOutput.java b/job/src/main/java/org/apache/kylin/job/common/HadoopCmdOutput.java
deleted file mode 100644
index 873607c..0000000
--- a/job/src/main/java/org/apache/kylin/job/common/HadoopCmdOutput.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.common;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TaskCounter;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class HadoopCmdOutput {
-
- protected static final Logger log = LoggerFactory.getLogger(HadoopCmdOutput.class);
-
- private final StringBuilder output;
- private final Job job;
-
- public HadoopCmdOutput(Job job, StringBuilder output) {
- super();
- this.job = job;
- this.output = output;
- }
-
- public String getMrJobId() {
- return getInfo().get(ExecutableConstants.MR_JOB_ID);
- }
-
- public Map<String, String> getInfo() {
- if (job != null) {
- Map<String, String> status = new HashMap<String, String>();
- if (null != job.getJobID()) {
- status.put(ExecutableConstants.MR_JOB_ID, job.getJobID().toString());
- }
- if (null != job.getTrackingURL()) {
- status.put(ExecutableConstants.YARN_APP_URL, job.getTrackingURL().toString());
- }
- return status;
- } else {
- return Collections.emptyMap();
- }
- }
-
- private String mapInputRecords;
- private String hdfsBytesWritten;
- private String hdfsBytesRead;
-
- public String getMapInputRecords() {
- return mapInputRecords;
- }
-
- public String getHdfsBytesWritten() {
- return hdfsBytesWritten;
- }
-
- public String getHdfsBytesRead() {
- return hdfsBytesRead;
- }
-
- public void updateJobCounter() {
- try {
- Counters counters = job.getCounters();
- if (counters == null) {
- String errorMsg = "no counters for job " + getMrJobId();
- log.warn(errorMsg);
- output.append(errorMsg);
- return;
- }
- this.output.append(counters.toString()).append("\n");
- log.debug(counters.toString());
-
- mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
- hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue());
- hdfsBytesRead = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_READ").getValue());
- } catch (Exception e) {
- log.error(e.getLocalizedMessage(), e);
- output.append(e.getLocalizedMessage());
-
- mapInputRecords = "0";
- hdfsBytesWritten = "0";
- hdfsBytesRead = "0";
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
deleted file mode 100644
index dc412ce..0000000
--- a/job/src/main/java/org/apache/kylin/job/common/HadoopShellExecutable.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.common;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.lang.reflect.Constructor;
-
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Created by qianzhou on 12/26/14.
- */
-public class HadoopShellExecutable extends AbstractExecutable {
-
- private static final String KEY_MR_JOB = "HADOOP_SHELL_JOB_CLASS";
- private static final String KEY_PARAMS = "HADOOP_SHELL_JOB_PARAMS";
-
- public HadoopShellExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final String mapReduceJobClass = getJobClass();
- String params = getJobParams();
- Preconditions.checkNotNull(mapReduceJobClass);
- Preconditions.checkNotNull(params);
- try {
- final Constructor<? extends AbstractHadoopJob> constructor = ClassUtil.forName(mapReduceJobClass, AbstractHadoopJob.class).getConstructor();
- final AbstractHadoopJob job = constructor.newInstance();
- String[] args = params.trim().split("\\s+");
- logger.info("parameters of the HadoopShellExecutable:");
- logger.info(params);
- int result;
- StringBuilder log = new StringBuilder();
- try {
- result = ToolRunner.run(job, args);
- } catch (Exception ex) {
- logger.error("error execute " + this.toString(), ex);
- StringWriter stringWriter = new StringWriter();
- ex.printStackTrace(new PrintWriter(stringWriter));
- log.append(stringWriter.toString()).append("\n");
- result = 2;
- }
- log.append("result code:").append(result);
- return result == 0 ? new ExecuteResult(ExecuteResult.State.SUCCEED, log.toString()) : new ExecuteResult(ExecuteResult.State.FAILED, log.toString());
- } catch (ReflectiveOperationException e) {
- logger.error("error getMapReduceJobClass, class name:" + getParam(KEY_MR_JOB), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- } catch (Exception e) {
- logger.error("error execute " + this.toString(), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setJobClass(Class<? extends AbstractHadoopJob> clazzName) {
- setParam(KEY_MR_JOB, clazzName.getName());
- }
-
- public String getJobClass() throws ExecuteException {
- return getParam(KEY_MR_JOB);
- }
-
- public void setJobParams(String param) {
- setParam(KEY_PARAMS, param);
- }
-
- public String getJobParams() {
- return getParam(KEY_PARAMS);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java b/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
deleted file mode 100644
index 75c461b..0000000
--- a/job/src/main/java/org/apache/kylin/job/common/HqlExecutable.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.common;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.HiveClient;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.datanucleus.store.types.backed.HashMap;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-
-/**
- * Created by qianzhou on 1/15/15.
- */
-public class HqlExecutable extends AbstractExecutable {
-
- private static final String HQL = "hql";
- private static final String HIVE_CONFIG = "hive-config";
-
- public HqlExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Map<String, String> configMap = getConfiguration();
- HiveClient hiveClient = new HiveClient(configMap);
-
- for (String hql : getHqls()) {
- hiveClient.executeHQL(hql);
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED);
- } catch (Exception e) {
- logger.error("error run hive query:" + getHqls(), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setConfiguration(Map<String, String> configMap) {
- if (configMap != null) {
- String configStr = "";
- try {
- configStr = JsonUtil.writeValueAsString(configMap);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- setParam(HIVE_CONFIG, configStr);
- }
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, String> getConfiguration() {
- String configStr = getParam(HIVE_CONFIG);
- Map<String, String> result = null;
- if (configStr != null) {
- try {
- result = JsonUtil.readValue(configStr, HashMap.class);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- return result;
- }
-
- public void setHqls(List<String> hqls) {
- setParam(HQL, StringUtils.join(hqls, ";"));
- }
-
- private List<String> getHqls() {
- final String hqls = getParam(HQL);
- if (hqls != null) {
- return Lists.newArrayList(StringUtils.split(hqls, ";"));
- } else {
- return Collections.emptyList();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
deleted file mode 100644
index cb6e76c..0000000
--- a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.common;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.lang.reflect.Constructor;
-import java.util.Collections;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Cluster;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.JobStatus;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.RMHAUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.job.execution.Output;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.tools.HadoopStatusChecker;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Created by qianzhou on 12/25/14.
- */
-public class MapReduceExecutable extends AbstractExecutable {
-
- public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
- private static final String KEY_MR_JOB = "MR_JOB_CLASS";
- private static final String KEY_PARAMS = "MR_JOB_PARAMS";
-
- public MapReduceExecutable() {
- super();
- }
-
- @Override
- protected void onExecuteStart(ExecutableContext executableContext) {
- final Output output = executableManager.getOutput(getId());
- if (output.getExtra().containsKey(START_TIME)) {
- final String mrJobId = output.getExtra().get(ExecutableConstants.MR_JOB_ID);
- if (mrJobId == null) {
- executableManager.updateJobOutput(getId(), ExecutableState.RUNNING, null, null);
- return;
- }
- try {
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- Job job = new Cluster(conf).getJob(JobID.forName(mrJobId));
- if (job.getJobState() == JobStatus.State.FAILED) {
- //remove previous mr job info
- super.onExecuteStart(executableContext);
- } else {
- executableManager.updateJobOutput(getId(), ExecutableState.RUNNING, null, null);
- }
- } catch (IOException e) {
- logger.warn("error get hadoop status");
- super.onExecuteStart(executableContext);
- } catch (InterruptedException e) {
- logger.warn("error get hadoop status");
- super.onExecuteStart(executableContext);
- }
- } else {
- super.onExecuteStart(executableContext);
- }
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final String mapReduceJobClass = getMapReduceJobClass();
- String params = getMapReduceParams();
- Preconditions.checkNotNull(mapReduceJobClass);
- Preconditions.checkNotNull(params);
- try {
- Job job;
- final Map<String, String> extra = executableManager.getOutput(getId()).getExtra();
- if (extra.containsKey(ExecutableConstants.MR_JOB_ID)) {
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- job = new Cluster(conf).getJob(JobID.forName(extra.get(ExecutableConstants.MR_JOB_ID)));
- logger.info("mr_job_id:" + extra.get(ExecutableConstants.MR_JOB_ID + " resumed"));
- } else {
- final Constructor<? extends AbstractHadoopJob> constructor = ClassUtil.forName(mapReduceJobClass, AbstractHadoopJob.class).getConstructor();
- final AbstractHadoopJob hadoopJob = constructor.newInstance();
- hadoopJob.setConf(HadoopUtil.getCurrentConfiguration());
- hadoopJob.setAsync(true); // so the ToolRunner.run() returns right away
- logger.info("parameters of the MapReduceExecutable:");
- logger.info(params);
- String[] args = params.trim().split("\\s+");
- try {
- //for async mr job, ToolRunner just return 0;
- ToolRunner.run(hadoopJob, args);
- } catch (Exception ex) {
- StringBuilder log = new StringBuilder();
- logger.error("error execute " + this.toString(), ex);
- StringWriter stringWriter = new StringWriter();
- ex.printStackTrace(new PrintWriter(stringWriter));
- log.append(stringWriter.toString()).append("\n");
- log.append("result code:").append(2);
- return new ExecuteResult(ExecuteResult.State.ERROR, log.toString());
- }
- job = hadoopJob.getJob();
- }
- final StringBuilder output = new StringBuilder();
- final HadoopCmdOutput hadoopCmdOutput = new HadoopCmdOutput(job, output);
-
- final String restStatusCheckUrl = getRestStatusCheckUrl(job, context.getConfig());
- if (restStatusCheckUrl == null) {
- logger.error("restStatusCheckUrl is null");
- return new ExecuteResult(ExecuteResult.State.ERROR, "restStatusCheckUrl is null");
- }
- String mrJobId = hadoopCmdOutput.getMrJobId();
- HadoopStatusChecker statusChecker = new HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, context.getConfig());
- JobStepStatusEnum status = JobStepStatusEnum.NEW;
- while (!isDiscarded()) {
- JobStepStatusEnum newStatus = statusChecker.checkStatus();
- if (status == JobStepStatusEnum.KILLED) {
- executableManager.updateJobOutput(getId(), ExecutableState.ERROR, Collections.<String, String> emptyMap(), "killed by admin");
- return new ExecuteResult(ExecuteResult.State.FAILED, "killed by admin");
- }
- if (status == JobStepStatusEnum.WAITING && (newStatus == JobStepStatusEnum.FINISHED || newStatus == JobStepStatusEnum.ERROR || newStatus == JobStepStatusEnum.RUNNING)) {
- final long waitTime = System.currentTimeMillis() - getStartTime();
- setMapReduceWaitTime(waitTime);
- }
- status = newStatus;
- executableManager.addJobInfo(getId(), hadoopCmdOutput.getInfo());
- if (status.isComplete()) {
- hadoopCmdOutput.updateJobCounter();
- final Map<String, String> info = hadoopCmdOutput.getInfo();
- info.put(ExecutableConstants.SOURCE_RECORDS_COUNT, hadoopCmdOutput.getMapInputRecords());
- info.put(ExecutableConstants.SOURCE_RECORDS_SIZE, hadoopCmdOutput.getHdfsBytesRead());
- info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hadoopCmdOutput.getHdfsBytesWritten());
- executableManager.addJobInfo(getId(), info);
-
- if (status == JobStepStatusEnum.FINISHED) {
- return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
- } else {
- return new ExecuteResult(ExecuteResult.State.FAILED, output.toString());
- }
- }
- Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000);
- }
-
- // try to kill running map-reduce job to release resources.
- if (job != null) {
- try {
- job.killJob();
- } catch (Exception e) {
- logger.warn("failed to kill hadoop job: " + job.getJobID(), e);
- }
- }
- return new ExecuteResult(ExecuteResult.State.DISCARDED, output.toString());
-
- } catch (ReflectiveOperationException e) {
- logger.error("error getMapReduceJobClass, class name:" + getParam(KEY_MR_JOB), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- } catch (Exception e) {
- logger.error("error execute " + this.toString(), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- private String getRestStatusCheckUrl(Job job, KylinConfig config) {
- final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
- if (yarnStatusCheckUrl != null) {
- return yarnStatusCheckUrl;
- } else {
- logger.info(KylinConfig.KYLIN_JOB_YARN_APP_REST_CHECK_URL + " is not set, read from job configuration");
- }
- String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, job.getConfiguration());
- if (HAUtil.isHAEnabled(job.getConfiguration())) {
- YarnConfiguration conf = new YarnConfiguration(job.getConfiguration());
- String active = RMHAUtils.findActiveRMHAId(conf);
- rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
- }
- if (StringUtils.isEmpty(rmWebHost)) {
- return null;
- }
- if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
- //do nothing
- } else {
- rmWebHost = "http://" + rmWebHost;
- }
- logger.info("yarn.resourcemanager.webapp.address:" + rmWebHost);
- return rmWebHost + "/ws/v1/cluster/apps/${job_id}?anonymous=true";
- }
-
- public long getMapReduceWaitTime() {
- return getExtraInfoAsLong(MAP_REDUCE_WAIT_TIME, 0L);
- }
-
- public void setMapReduceWaitTime(long t) {
- addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
- }
-
- public String getMapReduceJobClass() throws ExecuteException {
- return getParam(KEY_MR_JOB);
- }
-
- public void setMapReduceJobClass(Class<? extends AbstractHadoopJob> clazzName) {
- setParam(KEY_MR_JOB, clazzName.getName());
- }
-
- public String getMapReduceParams() {
- return getParam(KEY_PARAMS);
- }
-
- public void setMapReduceParams(String param) {
- setParam(KEY_PARAMS, param);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
deleted file mode 100644
index 19aa915..0000000
--- a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.common;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.common.util.Logger;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-import com.google.common.collect.Maps;
-
-/**
- * Created by qianzhou on 12/26/14.
- */
-public class ShellExecutable extends AbstractExecutable {
-
- private static final String CMD = "cmd";
-
- public ShellExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- logger.info("executing:" + getCmd());
- final ShellExecutableLogger logger = new ShellExecutableLogger();
- final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), logger);
- executableManager.addJobInfo(getId(), logger.getInfo());
- return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED, result.getSecond());
- } catch (IOException e) {
- logger.error("job:" + getId() + " execute finished with exception", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setCmd(String cmd) {
- setParam(CMD, cmd);
- }
-
- public String getCmd() {
- return getParam(CMD);
- }
-
- private static class ShellExecutableLogger implements Logger {
-
- private final Map<String, String> info = Maps.newHashMap();
-
- private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
- private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
- private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
- private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
-
- // hive
- private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
- private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
-
- @Override
- public void log(String message) {
- Matcher matcher = PATTERN_APP_ID.matcher(message);
- if (matcher.find()) {
- String appId = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_ID, appId);
- }
-
- matcher = PATTERN_APP_URL.matcher(message);
- if (matcher.find()) {
- String appTrackingUrl = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_URL, appTrackingUrl);
- }
-
- matcher = PATTERN_JOB_ID.matcher(message);
- if (matcher.find()) {
- String mrJobID = matcher.group(1);
- info.put(ExecutableConstants.MR_JOB_ID, mrJobID);
- }
-
- matcher = PATTERN_HDFS_BYTES_WRITTEN.matcher(message);
- if (matcher.find()) {
- String hdfsWritten = matcher.group(1);
- info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
- }
-
- matcher = PATTERN_SOURCE_RECORDS_COUNT.matcher(message);
- if (matcher.find()) {
- String sourceCount = matcher.group(1);
- info.put(ExecutableConstants.SOURCE_RECORDS_COUNT, sourceCount);
- }
-
- matcher = PATTERN_SOURCE_RECORDS_SIZE.matcher(message);
- if (matcher.find()) {
- String sourceSize = matcher.group(1);
- info.put(ExecutableConstants.SOURCE_RECORDS_SIZE, sourceSize);
- }
-
- // hive
- matcher = PATTERN_HIVE_APP_ID_URL.matcher(message);
- if (matcher.find()) {
- String jobId = matcher.group(1);
- String trackingUrl = matcher.group(2);
- info.put(ExecutableConstants.MR_JOB_ID, jobId);
- info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
- }
-
- matcher = PATTERN_HIVE_BYTES_WRITTEN.matcher(message);
- if (matcher.find()) {
- // String hdfsRead = matcher.group(1);
- String hdfsWritten = matcher.group(2);
- info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
- }
- }
-
- Map<String, String> getInfo() {
- return info;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
deleted file mode 100644
index 38f4a87..0000000
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.constant;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public interface BatchConstants {
-
- char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
-
- String CFG_CUBE_NAME = "cube.name";
- String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
- String CFG_CUBE_CUBOID_LEVEL = "cube.cuboid.level";
-
- String CFG_II_NAME = "ii.name";
- String CFG_II_SEGMENT_NAME = "ii.segment.name";
-
- String OUTPUT_PATH = "output.path";
-
- String TABLE_NAME = "table.name";
- String TABLE_COLUMNS = "table.columns";
-
- String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
-
- String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
-
- String MAPPER_SAMPLE_NUMBER = "mapper.sample.number";
- String REGION_NUMBER = "region.number";
- String REGION_NUMBER_MIN = "region.number.min";
- String REGION_NUMBER_MAX = "region.number.max";
- String REGION_SPLIT_SIZE = "region.split.size";
- String CUBE_CAPACITY = "cube.capacity";
-
- String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/";
- String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/";
-
- int COUNTER_MAX = 100000;
- int ERROR_RECORD_THRESHOLD = 100;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
deleted file mode 100644
index 3d98b0b..0000000
--- a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.constant;
-
-/**
- * Created by qianzhou on 1/5/15.
- */
-public final class ExecutableConstants {
-
- private ExecutableConstants() {
- }
-
- public static final String YARN_APP_ID = "yarn_application_id";
-
- public static final String YARN_APP_URL = "yarn_application_tracking_url";
- public static final String MR_JOB_ID = "mr_job_id";
- public static final String HDFS_BYTES_WRITTEN = "hdfs_bytes_written";
- public static final String SOURCE_RECORDS_COUNT = "source_records_count";
- public static final String SOURCE_RECORDS_SIZE = "source_records_size";
- public static final String GLOBAL_LISTENER_NAME = "ChainListener";
-
- public static final int DEFAULT_SCHEDULER_INTERVAL_SECONDS = 60;
-
- public static final String CUBE_JOB_GROUP_NAME = "cube_job_group";
-
- public static final String DAEMON_JOB_GROUP_NAME = "daemon_job_group";
- public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary";
-
- public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table";
- public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
- public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid Data";
- public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension Cuboid Data";
- public static final String STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION = "Calculate HTable Region Splits";
- public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
- public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert Cuboid Data to HFile";
- public static final String STEP_NAME_BULK_LOAD_HFILE = "Load HFile to HBase Table";
- public static final String STEP_NAME_MERGE_DICTIONARY = "Merge Cuboid Dictionary";
- public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
- public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
- public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
-
- public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
- public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
-
- public static final String PROP_ENGINE_CONTEXT = "jobengineConfig";
- public static final String PROP_JOB_FLOW = "jobFlow";
- public static final String PROP_JOBINSTANCE_UUID = "jobInstanceUuid";
- public static final String PROP_JOBSTEP_SEQ_ID = "jobStepSequenceID";
- public static final String PROP_COMMAND = "command";
- // public static final String PROP_STORAGE_LOCATION =
- // "storageLocationIdentifier";
- public static final String PROP_JOB_ASYNC = "jobAsync";
- public static final String PROP_JOB_CMD_EXECUTOR = "jobCmdExecutor";
- public static final String PROP_JOB_CMD_OUTPUT = "jobCmdOutput";
- public static final String PROP_JOB_KILLED = "jobKilled";
- public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows";
-
- public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java b/job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
deleted file mode 100644
index 9c8f083..0000000
--- a/job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.constant;
-
-public enum JobStatusEnum {
-
- NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16);
-
- private final int code;
-
- private JobStatusEnum(int statusCode) {
- this.code = statusCode;
- }
-
- public static JobStatusEnum getByCode(int statusCode) {
- for (JobStatusEnum status : values()) {
- if (status.getCode() == statusCode) {
- return status;
- }
- }
-
- return null;
- }
-
- public int getCode() {
- return this.code;
- }
-
- public boolean isComplete() {
- return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode() || code == JobStatusEnum.DISCARDED.getCode();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/constant/JobStepCmdTypeEnum.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/JobStepCmdTypeEnum.java b/job/src/main/java/org/apache/kylin/job/constant/JobStepCmdTypeEnum.java
deleted file mode 100644
index 0e4c18e..0000000
--- a/job/src/main/java/org/apache/kylin/job/constant/JobStepCmdTypeEnum.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.constant;
-
-/**
- * @author xduo, ysong1
- *
- */
-public enum JobStepCmdTypeEnum {
- SHELL_CMD, SHELL_CMD_HADOOP, JAVA_CMD_HADOOP_FACTDISTINCT, JAVA_CMD_HADOOP_BASECUBOID, JAVA_CMD_HADOOP_NDCUBOID, JAVA_CMD_HADOOP_RANGEKEYDISTRIBUTION, JAVA_CMD_HADOOP_CONVERTHFILE, JAVA_CMD_HADOOP_MERGECUBOID, JAVA_CMD_HADOOP_NO_MR_DICTIONARY, JAVA_CMD_HADDOP_NO_MR_CREATEHTABLE, JAVA_CMD_HADOOP_NO_MR_BULKLOAD
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java b/job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
deleted file mode 100644
index fbcfd97..0000000
--- a/job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.constant;
-
-public enum JobStepStatusEnum {
- NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16), WAITING(32), KILLED(64);
-
- private final int code;
-
- private JobStepStatusEnum(int statusCode) {
- this.code = statusCode;
- }
-
- public static JobStepStatusEnum getByCode(int statusCode) {
- for (JobStepStatusEnum status : values()) {
- if (status.getCode() == statusCode) {
- return status;
- }
- }
-
- return null;
- }
-
- public int getCode() {
- return this.code;
- }
-
- public boolean isComplete() {
- return code == JobStepStatusEnum.FINISHED.getCode() || code == JobStepStatusEnum.ERROR.getCode() || code == JobStepStatusEnum.DISCARDED.getCode();
- }
-
- public boolean isRunable() {
- return code == JobStepStatusEnum.PENDING.getCode() || code == JobStepStatusEnum.ERROR.getCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
deleted file mode 100644
index 182196c..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.job.common.MapReduceExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.job.execution.Output;
-
-/**
- * Created by qianzhou on 12/25/14.
- */
-public class CubingJob extends DefaultChainedExecutable {
-
- public CubingJob() {
- super();
- }
-
- private static final String CUBE_INSTANCE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
-
- public void setCubeName(String name) {
- setParam(CUBE_INSTANCE_NAME, name);
- }
-
- public String getCubeName() {
- return getParam(CUBE_INSTANCE_NAME);
- }
-
- void setSegmentIds(List<String> segmentIds) {
- setParam(SEGMENT_ID, StringUtils.join(segmentIds, ","));
- }
-
- void setSegmentId(String segmentId) {
- setParam(SEGMENT_ID, segmentId);
- }
-
- public String getSegmentIds() {
- return getParam(SEGMENT_ID);
- }
-
- @Override
- protected Pair<String, String> formatNotifications(ExecutableState state) {
- final Output output = jobService.getOutput(getId());
- String logMsg = "";
- switch (output.getState()) {
- case ERROR:
- logMsg = output.getVerboseMsg();
- break;
- case DISCARDED:
- break;
- case SUCCEED:
- break;
- default:
- return null;
- }
- String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
- content = content.replaceAll("\\$\\{job_name\\}", getName());
- content = content.replaceAll("\\$\\{result\\}", state.toString());
- content = content.replaceAll("\\$\\{cube_name\\}", getCubeName());
- content = content.replaceAll("\\$\\{source_records_count\\}", StringUtil.noBlank(getSourceRecordCount(), "0"));
- content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
- content = content.replaceAll("\\$\\{duration\\}", getDuration() / 60000 + "mins");
- content = content.replaceAll("\\$\\{mr_waiting\\}", getMapReduceWaitTime() / 60000 + "mins");
- content = content.replaceAll("\\$\\{last_update_time\\}", new Date(getLastModified()).toString());
- content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- content = content.replaceAll("\\$\\{error_log\\}", StringUtil.noBlank(logMsg, "no error log"));
-
- try {
- InetAddress inetAddress = InetAddress.getLocalHost();
- content = content.replaceAll("\\$\\{job_engine\\}", inetAddress.getCanonicalHostName());
- } catch (UnknownHostException e) {
- logger.warn(e.getLocalizedMessage(), e);
- }
-
- String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName();
- return Pair.of(title, content);
- }
-
- @Override
- protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
- long time = 0L;
- for (AbstractExecutable task : getTasks()) {
- final ExecutableState status = task.getStatus();
- if (status != ExecutableState.SUCCEED) {
- break;
- }
- if (task instanceof MapReduceExecutable) {
- time += ((MapReduceExecutable) task).getMapReduceWaitTime();
- }
- }
- setMapReduceWaitTime(time);
- super.onExecuteFinished(result, executableContext);
- }
-
- public long getMapReduceWaitTime() {
- return getExtraInfoAsLong(MAP_REDUCE_WAIT_TIME, 0L);
- }
-
- public void setMapReduceWaitTime(long t) {
- addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
- }
-
-
- public final String getSourceRecordCount() {
- for (AbstractExecutable task : getTasks()) {
- if (ExecutableConstants.STEP_NAME_BUILD_BASE_CUBOID.equals(task.getName())) {
- return getExtraInfo(task.getOutput(), ExecutableConstants.SOURCE_RECORDS_COUNT);
- }
- }
- return "N/A";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
deleted file mode 100644
index 80c030f..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ /dev/null
@@ -1,476 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.AbstractJobBuilder;
-import org.apache.kylin.job.common.HadoopShellExecutable;
-import org.apache.kylin.job.common.MapReduceExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.hadoop.cube.BaseCuboidJob;
-import org.apache.kylin.job.hadoop.cube.CubeHFileJob;
-import org.apache.kylin.job.hadoop.cube.FactDistinctColumnsJob;
-import org.apache.kylin.job.hadoop.cube.MergeCuboidJob;
-import org.apache.kylin.job.hadoop.cube.NDCuboidJob;
-import org.apache.kylin.job.hadoop.cube.RangeKeyDistributionJob;
-import org.apache.kylin.job.hadoop.dict.CreateDictionaryJob;
-import org.apache.kylin.job.hadoop.hbase.BulkLoadJob;
-import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
-import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Created by qianzhou on 12/25/14.
- */
-public final class CubingJobBuilder extends AbstractJobBuilder {
-
- public CubingJobBuilder(JobEngineConfig engineConfig) {
- super(engineConfig);
- }
-
- public CubingJob buildJob(CubeSegment seg) {
- checkPreconditions(seg);
-
- final CubingJob result = initialJob(seg, "BUILD");
- final String jobId = result.getId();
- final String cuboidRootPath = getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/cuboid/";
- final List<String> toDeletePaths = Lists.newArrayList();
-
- // cubing
- Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(seg, cuboidRootPath, result, toDeletePaths);
- String intermediateHiveTableStepId = twoSteps.getFirst().getId();
- String baseCuboidStepId = twoSteps.getSecond().getId();
-
- // convert htable
- AbstractExecutable convertCuboidToHfileStep = addHTableSteps(seg, cuboidRootPath, result);
-
- // update cube info
- result.addTask(createUpdateCubeInfoAfterBuildStep(seg, intermediateHiveTableStepId, baseCuboidStepId, convertCuboidToHfileStep.getId(), jobId));
-
- final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg);
- final String hiveIntermediateTable = this.getIntermediateHiveTableName(intermediateTableDesc, jobId);
- result.addTask(createGarbageCollectionStep(seg, null, hiveIntermediateTable, toDeletePaths));
-
- return result;
- }
-
- public CubingJob buildAndMergeJob(CubeSegment appendSegment, CubeSegment mergeSegment) {
- checkPreconditions(appendSegment, mergeSegment);
-
- CubingJob result = initialJob(mergeSegment, "BUILD");
- result.setSegmentIds(Lists.newArrayList(new String[] { appendSegment.getUuid(), mergeSegment.getUuid() }));
- final String jobId = result.getId();
- final String appendRootPath = getJobWorkingDir(jobId) + "/" + appendSegment.getCubeInstance().getName() + "/append_cuboid/";
- final String mergedRootPath = getJobWorkingDir(jobId) + "/" + appendSegment.getCubeInstance().getName() + "/cuboid/";
- List<String> mergingSegmentIds = Lists.newArrayList();
- List<String> mergingCuboidPaths = Lists.newArrayList();
- List<String> mergingHTables = Lists.newArrayList();
- final List<String> toDeletePaths = Lists.newArrayList();
-
- // cubing the incremental segment
- Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(appendSegment, appendRootPath, result, toDeletePaths);
- final String intermediateHiveTableStepId = twoSteps.getFirst().getId();
- final String baseCuboidStepId = twoSteps.getSecond().getId();
-
- // update the append segment info
- result.addTask(createUpdateCubeInfoAfterBuildStep(appendSegment, intermediateHiveTableStepId, baseCuboidStepId, null, jobId));
-
- List<CubeSegment> mergingSegments = mergeSegment.getCubeInstance().getMergingSegments(mergeSegment);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
-
- for (CubeSegment merging : mergingSegments) {
- mergingSegmentIds.add(merging.getUuid());
- mergingHTables.add(merging.getStorageLocationIdentifier());
- if (merging.equals(appendSegment)) {
- mergingCuboidPaths.add(appendRootPath + "*");
- } else {
- mergingCuboidPaths.add(getPathToMerge(merging));
- }
- toDeletePaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
- }
-
- // merge cuboid
- addMergeSteps(mergeSegment, mergingSegmentIds, mergingCuboidPaths, mergedRootPath, result);
-
- // convert htable
- AbstractExecutable convertCuboidToHfileStep = addHTableSteps(mergeSegment, mergedRootPath, result);
-
- // update cube info
- result.addTask(createUpdateCubeInfoAfterMergeStep(mergeSegment, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
- result.addTask(createGarbageCollectionStep(mergeSegment, mergingHTables, null, toDeletePaths));
-
- return result;
- }
-
- public CubingJob mergeJob(CubeSegment seg) {
- checkPreconditions(seg);
-
- CubingJob result = initialJob(seg, "MERGE");
- final String jobId = result.getId();
- final String mergedCuboidPath = getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/cuboid/";
-
- List<CubeSegment> mergingSegments = seg.getCubeInstance().getMergingSegments(seg);
- Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
- List<String> mergingSegmentIds = Lists.newArrayList();
- List<String> mergingCuboidPaths = Lists.newArrayList();
- List<String> mergingHTables = Lists.newArrayList();
- final List<String> toDeletePaths = Lists.newArrayList();
-
- for (CubeSegment merging : mergingSegments) {
- mergingSegmentIds.add(merging.getUuid());
- mergingCuboidPaths.add(getPathToMerge(merging));
- mergingHTables.add(merging.getStorageLocationIdentifier());
- toDeletePaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
- }
-
- // merge cuboid
- addMergeSteps(seg, mergingSegmentIds, mergingCuboidPaths, mergedCuboidPath, result);
-
- // convert htable
- AbstractExecutable convertCuboidToHfileStep = addHTableSteps(seg, mergedCuboidPath, result);
-
- // update cube info
- result.addTask(createUpdateCubeInfoAfterMergeStep(seg, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
- result.addTask(createGarbageCollectionStep(seg, mergingHTables, null, toDeletePaths));
- return result;
- }
-
- void addMergeSteps(CubeSegment seg, List<String> mergingSegmentIds, List<String> mergingCuboidPaths, String mergedCuboidPath, CubingJob result) {
-
- result.addTask(createMergeDictionaryStep(seg, mergingSegmentIds));
-
- String formattedPath = StringUtils.join(mergingCuboidPaths, ",");
- result.addTask(createMergeCuboidDataStep(seg, formattedPath, mergedCuboidPath));
- }
-
- Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result, List<String> toDeletePaths) {
- final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels();
- final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length;
-
- final String jobId = result.getId();
- final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg);
- final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId);
- final String intermediateHiveTableLocation = getIntermediateHiveTableLocation(intermediateTableDesc, jobId);
- final String factDistinctColumnsPath = getFactDistinctColumnsPath(seg, jobId);
- final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
-
- final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId);
- result.addTask(intermediateHiveTableStep);
-
- result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId));
-
- result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath));
-
- // base cuboid step
- final MapReduceExecutable baseCuboidStep = createBaseCuboidStep(seg, intermediateHiveTableLocation, cuboidOutputTempPath);
- result.addTask(baseCuboidStep);
-
- // n dim cuboid steps
- for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
- int dimNum = totalRowkeyColumnsCount - i;
- result.addTask(createNDimensionCuboidStep(seg, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount));
- }
-
- toDeletePaths.add(intermediateHiveTableLocation);
- toDeletePaths.add(factDistinctColumnsPath);
-
- return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep);
- }
-
- AbstractExecutable addHTableSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) {
- final String jobId = result.getId();
- final String cuboidPath = cuboidRootPath + "*";
-
- result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath, jobId));
- // create htable step
- result.addTask(createCreateHTableStep(seg, jobId));
- // generate hfiles step
- final MapReduceExecutable convertCuboidToHfileStep = createConvertCuboidToHfileStep(seg, cuboidPath, jobId);
- result.addTask(convertCuboidToHfileStep);
- // bulk load step
- result.addTask(createBulkLoadStep(seg, jobId));
-
- return convertCuboidToHfileStep;
- }
-
- private CubingJob initialJob(CubeSegment seg, String type) {
- CubingJob result = new CubingJob();
- SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone(engineConfig.getTimeZone()));
- result.setCubeName(seg.getCubeInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis())));
- result.setSubmitter(submitter);
- result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
- return result;
- }
-
- private void checkPreconditions(CubeSegment... segments) {
- for (CubeSegment seg : segments) {
- Preconditions.checkNotNull(seg, "segment cannot be null");
- }
- Preconditions.checkNotNull(engineConfig, "jobEngineConfig cannot be null");
- }
-
- private void appendMapReduceParameters(StringBuilder builder, CubeSegment seg) {
- try {
- String jobConf = engineConfig.getHadoopJobConfFilePath(seg.getCubeDesc().getModel().getCapacity());
- if (jobConf != null && jobConf.length() > 0) {
- builder.append(" -conf ").append(jobConf);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) {
- String[] paths = new String[groupRowkeyColumnsCount + 1];
- for (int i = 0; i <= groupRowkeyColumnsCount; i++) {
- int dimNum = totalRowkeyColumnCount - i;
- if (dimNum == totalRowkeyColumnCount) {
- paths[i] = cuboidRootPath + "base_cuboid";
- } else {
- paths[i] = cuboidRootPath + dimNum + "d_cuboid";
- }
- }
- return paths;
- }
-
- private String getPathToMerge(CubeSegment seg) {
- return getJobWorkingDir(seg.getLastBuildJobID()) + "/" + seg.getCubeInstance().getName() + "/cuboid/*";
- }
-
- private String getRowkeyDistributionOutputPath(CubeSegment seg, String jobId) {
- return HadoopUtil.makeQualifiedPathInHBaseCluster(getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats");
- }
-
- private String getFactDistinctColumnsPath(CubeSegment seg, String jobUuid) {
- return getJobWorkingDir(jobUuid) + "/" + seg.getCubeInstance().getName() + "/fact_distinct_columns";
- }
-
- private String getHFilePath(CubeSegment seg, String jobId) {
- return HadoopUtil.makeQualifiedPathInHBaseCluster(getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/hfile/");
- }
-
- private MapReduceExecutable createFactDistinctColumnsStep(CubeSegment seg, String intermediateHiveTableName, String jobId) {
- MapReduceExecutable result = new MapReduceExecutable();
- result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS);
- result.setMapReduceJobClass(FactDistinctColumnsJob.class);
- StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, seg);
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "output", getFactDistinctColumnsPath(seg, jobId));
- appendExecCmdParameters(cmd, "jobname", "Kylin_Fact_Distinct_Columns_" + seg.getCubeInstance().getName() + "_Step");
- appendExecCmdParameters(cmd, "tablename", intermediateHiveTableName);
-
- result.setMapReduceParams(cmd.toString());
- return result;
- }
-
- private HadoopShellExecutable createBuildDictionaryStep(CubeSegment seg, String factDistinctColumnsPath) {
- // base cuboid job
- HadoopShellExecutable buildDictionaryStep = new HadoopShellExecutable();
- buildDictionaryStep.setName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY);
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", factDistinctColumnsPath);
-
- buildDictionaryStep.setJobParams(cmd.toString());
- buildDictionaryStep.setJobClass(CreateDictionaryJob.class);
- return buildDictionaryStep;
- }
-
- private MapReduceExecutable createBaseCuboidStep(CubeSegment seg, String intermediateHiveTableLocation, String[] cuboidOutputTempPath) {
- // base cuboid job
- MapReduceExecutable baseCuboidStep = new MapReduceExecutable();
-
- StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, seg);
-
- baseCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_BASE_CUBOID);
-
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", intermediateHiveTableLocation);
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[0]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Base_Cuboid_Builder_" + seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "level", "0");
-
- baseCuboidStep.setMapReduceParams(cmd.toString());
- baseCuboidStep.setMapReduceJobClass(BaseCuboidJob.class);
- return baseCuboidStep;
- }
-
- private MapReduceExecutable createNDimensionCuboidStep(CubeSegment seg, String[] cuboidOutputTempPath, int dimNum, int totalRowkeyColumnCount) {
- // ND cuboid job
- MapReduceExecutable ndCuboidStep = new MapReduceExecutable();
-
- ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension");
- StringBuilder cmd = new StringBuilder();
-
- appendMapReduceParameters(cmd, seg);
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
- appendExecCmdParameters(cmd, "output", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
- appendExecCmdParameters(cmd, "jobname", "Kylin_ND-Cuboid_Builder_" + seg.getCubeInstance().getName() + "_Step");
- appendExecCmdParameters(cmd, "level", "" + (totalRowkeyColumnCount - dimNum));
-
- ndCuboidStep.setMapReduceParams(cmd.toString());
- ndCuboidStep.setMapReduceJobClass(NDCuboidJob.class);
- return ndCuboidStep;
- }
-
- private MapReduceExecutable createRangeRowkeyDistributionStep(CubeSegment seg, String inputPath, String jobId) {
- MapReduceExecutable rowkeyDistributionStep = new MapReduceExecutable();
- rowkeyDistributionStep.setName(ExecutableConstants.STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION);
- StringBuilder cmd = new StringBuilder();
-
- appendMapReduceParameters(cmd, seg);
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(seg, jobId));
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "jobname", "Kylin_Region_Splits_Calculator_" + seg.getCubeInstance().getName() + "_Step");
-
- rowkeyDistributionStep.setMapReduceParams(cmd.toString());
- rowkeyDistributionStep.setMapReduceJobClass(RangeKeyDistributionJob.class);
- return rowkeyDistributionStep;
- }
-
- private HadoopShellExecutable createCreateHTableStep(CubeSegment seg, String jobId) {
- HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
- createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg, jobId) + "/part-r-00000");
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
-
- createHtableStep.setJobParams(cmd.toString());
- createHtableStep.setJobClass(CreateHTableJob.class);
-
- return createHtableStep;
- }
-
- private MapReduceExecutable createConvertCuboidToHfileStep(CubeSegment seg, String inputPath, String jobId) {
- MapReduceExecutable createHFilesStep = new MapReduceExecutable();
- createHFilesStep.setName(ExecutableConstants.STEP_NAME_CONVERT_CUBOID_TO_HFILE);
- StringBuilder cmd = new StringBuilder();
-
- appendMapReduceParameters(cmd, seg);
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getHFilePath(seg, jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "jobname", "Kylin_HFile_Generator_" + seg.getCubeInstance().getName() + "_Step");
-
- createHFilesStep.setMapReduceParams(cmd.toString());
- createHFilesStep.setMapReduceJobClass(CubeHFileJob.class);
-
- return createHFilesStep;
- }
-
- private HadoopShellExecutable createBulkLoadStep(CubeSegment seg, String jobId) {
- HadoopShellExecutable bulkLoadStep = new HadoopShellExecutable();
- bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE);
-
- StringBuilder cmd = new StringBuilder();
- appendExecCmdParameters(cmd, "input", getHFilePath(seg, jobId));
- appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
-
- bulkLoadStep.setJobParams(cmd.toString());
- bulkLoadStep.setJobClass(BulkLoadJob.class);
-
- return bulkLoadStep;
-
- }
-
- private UpdateCubeInfoAfterBuildStep createUpdateCubeInfoAfterBuildStep(CubeSegment seg, String createFlatTableStepId, String baseCuboidStepId, String convertToHFileStepId, String jobId) {
- final UpdateCubeInfoAfterBuildStep updateCubeInfoStep = new UpdateCubeInfoAfterBuildStep();
- updateCubeInfoStep.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
- updateCubeInfoStep.setCubeName(seg.getCubeInstance().getName());
- updateCubeInfoStep.setSegmentId(seg.getUuid());
- updateCubeInfoStep.setCreateFlatTableStepId(createFlatTableStepId);
- updateCubeInfoStep.setBaseCuboidStepId(baseCuboidStepId);
- updateCubeInfoStep.setConvertToHFileStepId(convertToHFileStepId);
- updateCubeInfoStep.setCubingJobId(jobId);
- return updateCubeInfoStep;
- }
-
- private MergeDictionaryStep createMergeDictionaryStep(CubeSegment seg, List<String> mergingSegmentIds) {
- MergeDictionaryStep result = new MergeDictionaryStep();
- result.setName(ExecutableConstants.STEP_NAME_MERGE_DICTIONARY);
- result.setCubeName(seg.getCubeInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setMergingSegmentIds(mergingSegmentIds);
- return result;
- }
-
- private MapReduceExecutable createMergeCuboidDataStep(CubeSegment seg, String inputPath, String outputPath) {
- MapReduceExecutable mergeCuboidDataStep = new MapReduceExecutable();
- mergeCuboidDataStep.setName(ExecutableConstants.STEP_NAME_MERGE_CUBOID);
- StringBuilder cmd = new StringBuilder();
-
- appendMapReduceParameters(cmd, seg);
- appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "segmentname", seg.getName());
- appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", outputPath);
- appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + seg.getCubeInstance().getName() + "_Step");
-
- mergeCuboidDataStep.setMapReduceParams(cmd.toString());
- mergeCuboidDataStep.setMapReduceJobClass(MergeCuboidJob.class);
- return mergeCuboidDataStep;
- }
-
- private UpdateCubeInfoAfterMergeStep createUpdateCubeInfoAfterMergeStep(CubeSegment seg, List<String> mergingSegmentIds, String convertToHFileStepId, String jobId) {
- UpdateCubeInfoAfterMergeStep result = new UpdateCubeInfoAfterMergeStep();
- result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
- result.setCubeName(seg.getCubeInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setMergingSegmentIds(mergingSegmentIds);
- result.setConvertToHFileStepId(convertToHFileStepId);
- result.setCubingJobId(jobId);
- return result;
- }
-
- private GarbageCollectionStep createGarbageCollectionStep(CubeSegment seg, List<String> oldHtables, String hiveIntermediateTable, List<String> oldHdsfPaths) {
- GarbageCollectionStep result = new GarbageCollectionStep();
- result.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
- result.setOldHTables(oldHtables);
- result.setOldHiveTable(hiveIntermediateTable);
- result.setOldHdfsPaths(oldHdsfPaths);
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java b/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
deleted file mode 100644
index f2f1fc0..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.cmd.ShellCmdOutput;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * Drop the resources that is no longer needed, including intermediate hive table (after cube build) and hbase tables (after cube merge)
- */
-public class GarbageCollectionStep extends AbstractExecutable {
-
- private static final String OLD_HTABLES = "oldHTables";
-
- private static final String OLD_HIVE_TABLE = "oldHiveTable";
-
- private static final String OLD_HDFS_PATHS = "oldHdfsPaths";
-
- private static final Logger logger = LoggerFactory.getLogger(GarbageCollectionStep.class);
-
- private StringBuffer output;
-
- public GarbageCollectionStep() {
- super();
- output = new StringBuffer();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
-
- try {
- dropHBaseTable(context);
- dropHiveTable(context);
- dropHdfsPath(context);
- } catch (IOException e) {
- logger.error("job:" + getId() + " execute finished with exception", e);
- output.append("\n").append(e.getLocalizedMessage());
- return new ExecuteResult(ExecuteResult.State.ERROR, output.toString());
- }
-
- return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
- }
-
- private void dropHiveTable(ExecutableContext context) throws IOException {
- final String hiveTable = this.getOldHiveTable();
- if (StringUtils.isNotEmpty(hiveTable)) {
- final String dropSQL = "USE " + KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() + ";" + " DROP TABLE IF EXISTS " + hiveTable + ";";
- final String dropHiveCMD = "hive -e \"" + dropSQL + "\"";
- logger.info("executing: " + dropHiveCMD);
- ShellCmdOutput shellCmdOutput = new ShellCmdOutput();
- context.getConfig().getCliCommandExecutor().execute(dropHiveCMD, shellCmdOutput);
- logger.debug("Dropped Hive table " + hiveTable + " \n");
- output.append(shellCmdOutput.getOutput() + " \n");
- output.append("Dropped Hive table " + hiveTable + " \n");
- }
-
- }
-
- private void dropHBaseTable(ExecutableContext context) throws IOException {
- List<String> oldTables = getOldHTables();
- if (oldTables != null && oldTables.size() > 0) {
- String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = null;
- try {
- admin = new HBaseAdmin(conf);
- for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
- String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
- if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
- }
- admin.deleteTable(table);
- logger.debug("Dropped HBase table " + table);
- output.append("Dropped HBase table " + table + " \n");
- } else {
- logger.debug("Skipped HBase table " + table);
- output.append("Skipped HBase table " + table + " \n");
- }
- }
- }
-
- } finally {
- if (admin != null)
- try {
- admin.close();
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage());
- }
- }
- }
- }
-
- private void dropHdfsPathOnCluster(List<String> oldHdfsPaths, FileSystem fileSystem) throws IOException {
- if (oldHdfsPaths != null && oldHdfsPaths.size() > 0) {
- logger.debug("Drop HDFS path on FileSystem: " + fileSystem.getUri());
- output.append("Drop HDFS path on FileSystem: \"" + fileSystem.getUri() + "\" \n");
- for (String path : oldHdfsPaths) {
- if (path.endsWith("*"))
- path = path.substring(0, path.length() - 1);
-
- Path oldPath = new Path(path);
- if (fileSystem.exists(oldPath)) {
- fileSystem.delete(oldPath, true);
- logger.debug("Dropped HDFS path: " + path);
- output.append("Dropped HDFS path \"" + path + "\" \n");
- } else {
- logger.debug("HDFS path not exists: " + path);
- output.append("HDFS path not exists: \"" + path + "\" \n");
- }
- }
- }
- }
-
- private void dropHdfsPath(ExecutableContext context) throws IOException {
- List<String> oldHdfsPaths = this.getOldHdfsPaths();
- FileSystem fileSystem = FileSystem.get(HadoopUtil.getCurrentConfiguration());
- dropHdfsPathOnCluster(oldHdfsPaths, fileSystem);
-
- if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) {
- fileSystem = FileSystem.get(HadoopUtil.getCurrentHBaseConfiguration());
- dropHdfsPathOnCluster(oldHdfsPaths, fileSystem);
- }
-
- }
-
- public void setOldHTables(List<String> tables) {
- setArrayParam(OLD_HTABLES, tables);
- }
-
- private List<String> getOldHTables() {
- return getArrayParam(OLD_HTABLES);
- }
-
- public void setOldHdfsPaths(List<String> paths) {
- setArrayParam(OLD_HDFS_PATHS, paths);
- }
-
- private List<String> getOldHdfsPaths() {
- return getArrayParam(OLD_HDFS_PATHS);
- }
-
- private void setArrayParam(String paramKey, List<String> paramValues) {
- setParam(paramKey, StringUtils.join(paramValues, ","));
- }
-
- private List<String> getArrayParam(String paramKey) {
- final String ids = getParam(paramKey);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
- public void setOldHiveTable(String hiveTable) {
- setParam(OLD_HIVE_TABLE, hiveTable);
- }
-
- private String getOldHiveTable() {
- return getParam(OLD_HIVE_TABLE);
- }
-
-}
[28/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv b/examples/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv
deleted file mode 100644
index 67efe23..0000000
--- a/examples/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv
+++ /dev/null
@@ -1,731 +0,0 @@
-2012-08-16,2012-01-01,2012-07-01,2012-08-01,2012-08-11,0,-1,-3,-15,-103,0,-1,-4,-15,-15,41501,228,47,16,6,33,5928,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-17,16-Aug-2012,Aug 16th 2012,Fri 08-16-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-11,2012-08-12,2012-08-16,Fri ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,33,2012,2012-08-11,2012-08-17,N,Wk.33 - 13,2012-08-11 00:00:00,2012-08-17 00:00:00,2012W33 ,2012W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2012,N,2012-08-16,2011-08-16,2012-05-16,2012-02-16,2012-07-16,2012-06-16,2012-08-09,2012-08-02,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-03,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-328,0,-3,-11,-47,-47,41276,3,3,3,5,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,03-Jan-2012,Jan 3rd 2012,Thu 01-03-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-03,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-03,2011-01-03,2012-10-03,2012-07-03,2012-12-03,2012-11-03,2012-12-27,2012-12-20,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-10,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-231,0,-2,-8,-33,-33,41373,100,10,10,4,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,10-Apr-2012,Apr 10th 2012,Wed 04-10-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-10,Wed ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-10,2011-04-10,2012-01-10,2012-10-10,2012-03-10,2012-02-10,2012-04-03,2012-03-27,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-12,2012-01-01,2012-04-01,2012-06-01,2012-06-09,0,-2,-5,-24,-168,0,-2,-6,-24,-24,41436,163,73,12,4,24,5919,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-15,12-Jun-2012,Jun 12th 2012,Wed 06-12-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-09,2012-06-10,2012-06-12,Wed ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,24,2012,2012-06-09,2012-06-15,N,Wk.24 - 13,2012-06-09 00:00:00,2012-06-15 00:00:00,2012W24 ,2012W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2012,N,2012-06-12,2011-06-12,2012-03-12,2012-12-12,2012-05-12,2012-04-12,2012-06-05,2012-05-29,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-27,2012-01-01,2012-04-01,2012-04-01,2012-04-21,0,-2,-7,-31,-214,0,-2,-8,-31,-31,41390,117,27,27,7,17,5912,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-27,27-Apr-2012,Apr 27th 2012,Sat 04-27-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-21,2012-04-22,2012-04-27,Sat ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,17,2012,2012-04-21,2012-04-27,N,Wk.17 - 13,2012-04-21 00:00:00,2012-04-27 00:00:00,2012W17 ,2012W17 ,04/21/13 - 04/27/13,04/21 - 04/27,2012,N,2012-04-27,2011-04-27,2012-01-27,2012-10-27,2012-03-27,2012-02-27,2012-04-20,2012-04-13,0,0,0,0,0,0,0,0,4,2,17,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-19,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-69,0,-1,-3,-10,-10,41535,262,81,19,5,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,19-Sep-2012,Sep 19th 2012,Thu 09-19-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-19,Thu ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-19,2011-09-19,2012-06-19,2012-03-19,2012-08-19,2012-07-19,2012-09-12,2012-09-05,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-07,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-265,0,-3,-9,-38,-38,41339,66,66,7,5,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,07-Mar-2012,Mar 7th 2012,Thu 03-07-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-07,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-07,2011-03-07,2012-12-07,2012-09-07,2012-02-07,2012-01-07,2012-02-28,2012-02-21,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-28,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-244,0,-3,-9,-35,-35,41360,87,87,28,5,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,28-Mar-2012,Mar 28th 2012,Thu 03-28-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-28,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-28,2011-03-28,2012-12-28,2012-09-28,2012-02-28,2012-01-28,2012-03-21,2012-03-14,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-23,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-4,0,0,-1,-1,-1,41600,327,54,23,7,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,23-Nov-2012,Nov 23rd 2012,Sat 11-23-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-23,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-23,2011-11-23,2012-08-23,2012-05-23,2012-10-23,2012-09-23,2012-11-16,2012-11-09,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-09,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-263,0,-3,-9,-38,-38,41341,68,68,9,7,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,09-Mar-2012,Mar 9th 2012,Sat 03-09-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-09,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-09,2011-03-09,2012-12-09,2012-09-09,2012-02-09,2012-01-09,2012-03-02,2012-02-23,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-21,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-6,0,0,-1,-1,-1,41598,325,52,21,5,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,21-Nov-2012,Nov 21st 2012,Thu 11-21-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-21,Thu ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-21,2011-11-21,2012-08-21,2012-05-21,2012-10-21,2012-09-21,2012-11-14,2012-11-07,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-26,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-246,0,-3,-9,-35,-35,41358,85,85,26,3,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,26-Mar-2012,Mar 26th 2012,Tue 03-26-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-26,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-26,2011-03-26,2012-12-26,2012-09-26,2012-02-26,2012-01-26,2012-03-19,2012-03-12,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-24,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-307,0,-3,-11,-44,-44,41297,24,24,24,5,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,24-Jan-2012,Jan 24th 2012,Thu 01-24-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-24,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-24,2011-01-24,2012-10-24,2012-07-24,2012-12-24,2012-11-24,2012-01-17,2012-01-10,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-01,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-330,0,-3,-11,-47,-47,41274,1,1,1,3,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,01-Jan-2012,Jan 1st 2012,Tue 01-01-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-01,Tue ,2012M01,Jan-2012,Y,2012M01 ,Y,Year 2012 - Quarter 01,2012Q01 ,Y,1,2012,2012-12-30,2012-01-05,Y,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,Y,2012-01-01,2011-01-01,2012-10-01,2012-07-01,2012-12-01,2012-11-01,2012-12-25,2012-12-18,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-29,2012-01-01,2012-04-01,2012-04-01,2012-04-28,0,-2,-7,-30,-212,0,-2,-7,-30,-30,41392,119,29,29,2,18,5913,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-05-04,29-Apr-2012,Apr 29th 2012,Mon 04-29-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-04-28,2012-04-28,2012-04-29,2012-04-29,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,18,2012,2012-04-28,2012-05-04,N,Wk.18 - 13,2012-04-28 00:00:00,2012-05-04 00:00:00,2012W18 ,2012W18 ,04/28/13 - 05/04/13,04/28 - 05/04,2012,N,2012-04-29,2011-04-29,2012-01-29,2012-10-29,2012-03-29,2012-02-28,2012-04-22,2012-04-15,0,0,0,0,0,0,0,0,5,2,18,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-11,2012-01-01,2012-04-01,2012-05-01,2012-05-05,0,-2,-6,-29,-200,0,-2,-7,-29,-29,41404,131,41,11,7,19,5914,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-11,11-May-2012,May 11th 2012,Sat 05-11-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-05,2012-05-06,2012-05-11,Sat ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,19,2012,2012-05-05,2012-05-11,N,Wk.19 - 13,2012-05-05 00:00:00,2012-05-11 00:00:00,2012W19 ,2012W19 ,05/05/13 - 05/11/13,05/05 - 05/11,2012,N,2012-05-11,2011-05-11,2012-02-11,2012-11-11,2012-04-11,2012-03-11,2012-05-04,2012-04-27,0,0,0,0,0,0,0,0,5,2,19,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-01,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-57,0,0,-2,-8,-8,41547,274,1,1,3,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,01-Oct-2012,Oct 1st 2012,Tue 10-01-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-01,Tue ,2012M10,Oct-2012,Y,2012M10 ,Y,Year 2012 - Quarter 04,2012Q04 ,Y,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-01,2011-10-01,2012-07-01,2012-04-01,2012-09-01,2012-08-01,2012-09-24,2012-09-17,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-05,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,8,0,0,0,1,1,41612,339,66,5,5,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,05-Dec-2012,Dec 5th 2012,Thu 12-05-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-05,Thu ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-05,2011-12-05,2012-09-05,2012-06-05,2012-11-05,2012-10-05,2012-11-28,2012-11-21,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-05,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-267,0,-3,-9,-38,-38,41337,64,64,5,3,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,05-Mar-2012,Mar 5th 2012,Tue 03-05-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-05,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-05,2011-03-05,2012-12-05,2012-09-05,2012-02-05,2012-01-05,2012-02-26,2012-02-19,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-03,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-55,0,0,-2,-8,-8,41549,276,3,3,5,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,03-Oct-2012,Oct 3rd 2012,Thu 10-03-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-03,Thu ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-03,2011-10-03,2012-07-03,2012-04-03,2012-09-03,2012-08-03,2012-09-26,2012-09-19,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-22,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-309,0,-3,-11,-44,-44,41295,22,22,22,3,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,22-Jan-2012,Jan 22nd 2012,Tue 01-22-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-22,Tue ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-22,2011-01-22,2012-10-22,2012-07-22,2012-12-22,2012-11-22,2012-01-15,2012-01-08,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-26,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,29,0,0,0,4,4,41633,360,87,26,5,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,26-Dec-2012,Dec 26th 2012,Thu 12-26-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-26,Thu ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-26,2011-12-26,2012-09-26,2012-06-26,2012-11-26,2012-10-26,2012-12-19,2012-12-12,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-17,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-71,0,-1,-3,-10,-10,41533,260,79,17,3,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,17-Sep-2012,Sep 17th 2012,Tue 09-17-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-17,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-17,2011-09-17,2012-06-17,2012-03-17,2012-08-17,2012-07-17,2012-09-10,2012-09-03,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-20,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-38,0,0,-2,-5,-6,41566,293,20,20,1,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,20-Oct-2012,Oct 20th 2012,Sun 10-20-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-14,2012-10-20,Sun ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,Y,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-20,2011-10-20,2012-07-20,2012-04-20,2012-09-20,2012-08-20,2012-10-13,2012-10-06,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-25,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,-2,0,0,0,0,0,41602,329,56,25,2,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,25-Nov-2012,Nov 25th 2012,Mon 11-25-13,1,1,1,1,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-25,2012-11-25,Mon ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,N,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-25,2011-11-25,2012-08-25,2012-05-25,2012-10-25,2012-09-25,2012-11-18,2012-11-11,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-26,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-305,0,-3,-11,-44,-44,41299,26,26,26,7,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,26-Jan-2012,Jan 26th 2012,Sat 01-26-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-26,Sat ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-26,2011-01-26,2012-10-26,2012-07-26,2012-12-26,2012-11-26,2012-01-19,2012-01-12,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-24,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,27,0,0,0,4,4,41631,358,85,24,3,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,24-Dec-2012,Dec 24th 2012,Tue 12-24-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-24,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-24,2011-12-24,2012-09-24,2012-06-24,2012-11-24,2012-10-24,2012-12-17,2012-12-10,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-04,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-296,0,-3,-10,-42,-42,41308,35,35,4,2,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,04-Feb-2012,Feb 4th 2012,Mon 02-04-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-04,Mon ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-04,2011-02-04,2012-11-04,2012-08-04,2012-01-04,2012-12-04,2012-01-28,2012-01-21,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-30,2012-01-01,2012-04-01,2012-05-01,2012-05-26,0,-2,-6,-26,-181,0,-2,-6,-26,-26,41423,150,60,30,5,22,5917,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-06-01,30-May-2012,May 30th 2012,Thu 05-30-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-05-26,2012-05-26,2012-05-27,2012-05-30,Thu ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,22,2012,2012-05-26,2012-06-01,N,Wk.22 - 13,2012-05-26 00:00:00,2012-06-01 00:00:00,2012W22 ,2012W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2012,N,2012-05-30,2011-05-30,2012-02-28,2012-11-30,2012-04-30,2012-03-30,2012-05-23,2012-05-16,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-12,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-229,0,-2,-8,-33,-33,41375,102,12,12,6,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,12-Apr-2012,Apr 12th 2012,Fri 04-12-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-12,Fri ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-12,2011-04-12,2012-01-12,2012-10-12,2012-03-12,2012-02-12,2012-04-05,2012-03-29,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-08,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-233,0,-2,-8,-33,-33,41371,98,8,8,2,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,08-Apr-2012,Apr 8th 2012,Mon 04-08-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-08,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-08,2011-04-08,2012-01-08,2012-10-08,2012-03-08,2012-02-08,2012-04-01,2012-03-25,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-28,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,31,0,0,0,4,4,41635,362,89,28,7,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,28-Dec-2012,Dec 28th 2012,Sat 12-28-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-28,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-28,2011-12-28,2012-09-28,2012-06-28,2012-11-28,2012-10-28,2012-12-21,2012-12-14,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-03,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,6,0,0,0,1,1,41610,337,64,3,3,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,03-Dec-2012,Dec 3rd 2012,Tue 12-03-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-03,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-03,2011-12-03,2012-09-03,2012-06-03,2012-11-03,2012-10-03,2012-11-26,2012-11-19,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-15,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-135,0,-1,-5,-19,-19,41469,196,15,15,2,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,15-Jul-2012,Jul 15th 2012,Mon 07-15-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-15,Mon ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-15,2011-07-15,2012-04-15,2012-01-15,2012-06-15,2012-05-15,2012-07-08,2012-07-01,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-13,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-198,0,-2,-7,-28,-28,41406,133,43,13,2,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,13-May-2012,May 13th 2012,Mon 05-13-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-13,Mon ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-13,2011-05-13,2012-02-13,2012-11-13,2012-04-13,2012-03-13,2012-05-06,2012-04-29,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-30,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-242,0,-3,-9,-35,-35,41362,89,89,30,7,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,30-Mar-2012,Mar 30th 2012,Sat 03-30-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-30,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-30,2011-03-30,2012-12-30,2012-09-30,2012-02-28,2012-01-30,2012-03-23,2012-03-16,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-09,2012-01-01,2012-10-01,2012-12-01,2012-12-08,0,0,1,2,12,0,0,0,2,2,41616,343,70,9,2,50,5945,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-14,09-Dec-2012,Dec 9th 2012,Mon 12-09-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-08,2012-12-09,2012-12-09,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,50,2012,2012-12-08,2012-12-14,N,Wk.50 - 13,2012-12-08 00:00:00,2012-12-14 00:00:00,2012W50 ,2012W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2012,N,2012-12-09,2011-12-09,2012-09-09,2012-06-09,2012-11-09,2012-10-09,2012-12-02,2012-11-25,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-25,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-275,0,-3,-9,-39,-39,41329,56,56,25,2,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,25-Feb-2012,Feb 25th 2012,Mon 02-25-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-02-25,Mon ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-02-25,2011-02-25,2012-11-25,2012-08-25,2012-01-25,2012-12-25,2012-02-18,2012-02-11,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-04,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-23,0,0,-1,-3,-3,41581,308,35,4,2,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,04-Nov-2012,Nov 4th 2012,Mon 11-04-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-04,Mon ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-04,2011-11-04,2012-08-04,2012-05-04,2012-10-04,2012-09-04,2012-10-28,2012-10-21,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-08,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-19,0,0,-1,-3,-3,41585,312,39,8,6,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,08-Nov-2012,Nov 8th 2012,Fri 11-08-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-08,Fri ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-08,2011-11-08,2012-08-08,2012-05-08,2012-10-08,2012-09-08,2012-11-01,2012-10-25,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-22,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-97,0,-1,-4,-14,-14,41507,234,53,22,5,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,22-Aug-2012,Aug 22nd 2012,Thu 08-22-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-22,Thu ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-22,2011-08-22,2012-05-22,2012-02-22,2012-07-22,2012-06-22,2012-08-15,2012-08-08,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-18,2012-01-01,2012-04-01,2012-06-01,2012-06-16,0,-2,-5,-23,-162,0,-2,-6,-23,-23,41442,169,79,18,3,25,5920,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-22,18-Jun-2012,Jun 18th 2012,Tue 06-18-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-16,2012-06-17,2012-06-18,Tue ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,25,2012,2012-06-16,2012-06-22,N,Wk.25 - 13,2012-06-16 00:00:00,2012-06-22 00:00:00,2012W25 ,2012W25 ,06/16/13 - 06/22/13,06/16 - 06/22,2012,N,2012-06-18,2011-06-18,2012-03-18,2012-12-18,2012-05-18,2012-04-18,2012-06-11,2012-06-04,0,0,0,0,0,0,0,0,6,2,25,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-19,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-131,0,-1,-5,-19,-19,41473,200,19,19,6,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,19-Jul-2012,Jul 19th 2012,Fri 07-19-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-19,Fri ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-19,2011-07-19,2012-04-19,2012-01-19,2012-06-19,2012-05-19,2012-07-12,2012-07-05,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-20,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-99,0,-1,-4,-14,-14,41505,232,51,20,3,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,20-Aug-2012,Aug 20th 2012,Tue 08-20-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-20,Tue ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-20,2011-08-20,2012-05-20,2012-02-20,2012-07-20,2012-06-20,2012-08-13,2012-08-06,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-24,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-95,0,-1,-4,-14,-14,41509,236,55,24,7,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,24-Aug-2012,Aug 24th 2012,Sat 08-24-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-24,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-24,2011-08-24,2012-05-24,2012-02-24,2012-07-24,2012-06-24,2012-08-17,2012-08-10,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-13,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-137,0,-1,-5,-20,-20,41467,194,13,13,7,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,13-Jul-2012,Jul 13th 2012,Sat 07-13-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-13,Sat ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-13,2011-07-13,2012-04-13,2012-01-13,2012-06-13,2012-05-13,2012-07-06,2012-06-29,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-06,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-21,0,0,-1,-3,-3,41583,310,37,6,4,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,06-Nov-2012,Nov 6th 2012,Wed 11-06-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-06,Wed ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-06,2011-11-06,2012-08-06,2012-05-06,2012-10-06,2012-09-06,2012-10-30,2012-10-23,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-07,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,10,0,0,0,1,1,41614,341,68,7,7,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,07-Dec-2012,Dec 7th 2012,Sat 12-07-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-07,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-07,2011-12-07,2012-09-07,2012-06-07,2012-11-07,2012-10-07,2012-11-30,2012-11-23,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-04,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-146,0,-1,-5,-21,-21,41458,185,4,4,5,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,04-Jul-2012,Jul 4th 2012,Thu 07-04-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-04,Thu ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-04,2011-07-04,2012-04-04,2012-01-04,2012-06-04,2012-05-04,2012-06-27,2012-06-20,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-07,2012-01-01,2012-10-01,2012-10-01,2012-10-06,0,0,-1,-7,-51,0,0,-2,-7,-7,41553,280,7,7,2,41,5936,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-12,07-Oct-2012,Oct 7th 2012,Mon 10-07-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-06,2012-10-07,2012-10-07,Mon ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,41,2012,2012-10-06,2012-10-12,N,Wk.41 - 13,2012-10-06 00:00:00,2012-10-12 00:00:00,2012W41 ,2012W41 ,10/06/13 - 10/12/13,10/06 - 10/12,2012,N,2012-10-07,2011-10-07,2012-07-07,2012-04-07,2012-09-07,2012-08-07,2012-09-30,2012-09-23,0,0,0,0,0,0,0,0,10,4,41,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-10,2012-01-01,2012-10-01,2012-11-01,2012-11-10,0,0,0,-2,-17,0,0,-1,-2,-3,41587,314,41,10,1,46,5941,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-16,10-Nov-2012,Nov 10th 2012,Sun 11-10-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-10,2012-11-04,2012-11-10,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,46,2012,2012-11-10,2012-11-16,Y,Wk.46 - 13,2012-11-10 00:00:00,2012-11-16 00:00:00,2012W46 ,2012W46 ,11/10/13 - 11/16/13,11/10 - 11/16,2012,N,2012-11-10,2011-11-10,2012-08-10,2012-05-10,2012-10-10,2012-09-10,2012-11-03,2012-10-27,0,0,0,0,0,0,0,0,11,4,46,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-07,2012-01-01,2012-07-01,2012-08-01,2012-08-04,0,-1,-3,-16,-112,0,-1,-4,-16,-16,41492,219,38,7,4,32,5927,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-10,07-Aug-2012,Aug 7th 2012,Wed 08-07-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-04,2012-08-05,2012-08-07,Wed ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,32,2012,2012-08-04,2012-08-10,N,Wk.32 - 13,2012-08-04 00:00:00,2012-08-10 00:00:00,2012W32 ,2012W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2012,N,2012-08-07,2011-08-07,2012-05-07,2012-02-07,2012-07-07,2012-06-07,2012-07-31,2012-07-24,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-25,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-63,0,-1,-3,-9,-9,41541,268,87,25,4,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,25-Sep-2012,Sep 25th 2012,Wed 09-25-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-23,2012-09-25,Wed ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,N,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-25,2011-09-25,2012-06-25,2012-03-25,2012-08-25,2012-07-25,2012-09-18,2012-09-11,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-05,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-53,0,0,-2,-8,-8,41551,278,5,5,7,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,05-Oct-2012,Oct 5th 2012,Sat 10-05-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-05,Sat ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-05,2011-10-05,2012-07-05,2012-04-05,2012-09-05,2012-08-05,2012-09-28,2012-09-21,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-02,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-86,0,-1,-3,-12,-12,41518,245,64,2,2,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,02-Sep-2012,Sep 2nd 2012,Mon 09-02-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-09-02,2012-09-02,Mon ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,N,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-02,2011-09-02,2012-06-02,2012-03-02,2012-08-02,2012-07-02,2012-08-26,2012-08-19,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-15,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-196,0,-2,-7,-28,-28,41408,135,45,15,4,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,15-May-2012,May 15th 2012,Wed 05-15-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-15,Wed ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-15,2011-05-15,2012-02-15,2012-11-15,2012-04-15,2012-03-15,2012-05-08,2012-05-01,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-10,2012-01-01,2012-01-01,2012-02-01,2012-02-10,0,-3,-9,-41,-290,0,-3,-10,-41,-42,41314,41,41,10,1,7,5902,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-16,10-Feb-2012,Feb 10th 2012,Sun 02-10-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-10,2012-02-04,2012-02-10,Sun ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,7,2012,2012-02-10,2012-02-16,Y,Wk.07 - 13,2012-02-10 00:00:00,2012-02-16 00:00:00,2012W07 ,2012W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2012,N,2012-02-10,2011-02-10,2012-11-10,2012-08-10,2012-01-10,2012-12-10,2012-02-03,2012-01-27,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-21,2012-01-01,2012-10-01,2012-12-01,2012-12-15,0,0,1,3,24,0,0,0,3,3,41628,355,82,21,7,51,5946,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-21,21-Dec-2012,Dec 21st 2012,Sat 12-21-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-15,2012-12-16,2012-12-21,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,51,2012,2012-12-15,2012-12-21,N,Wk.51 - 13,2012-12-15 00:00:00,2012-12-21 00:00:00,2012W51 ,2012W51 ,12/15/13 - 12/21/13,12/15 - 12/21,2012,N,2012-12-21,2011-12-21,2012-09-21,2012-06-21,2012-11-21,2012-10-21,2012-12-14,2012-12-07,0,0,0,0,0,0,0,0,12,4,51,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-22,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-36,0,0,-2,-5,-5,41568,295,22,22,3,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,22-Oct-2012,Oct 22nd 2012,Tue 10-22-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-22,Tue ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-22,2011-10-22,2012-07-22,2012-04-22,2012-09-22,2012-08-22,2012-10-15,2012-10-08,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-24,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-34,0,0,-2,-5,-5,41570,297,24,24,5,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,24-Oct-2012,Oct 24th 2012,Thu 10-24-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-24,Thu ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-24,2011-10-24,2012-07-24,2012-04-24,2012-09-24,2012-08-24,2012-10-17,2012-10-10,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-10,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-140,0,-1,-5,-20,-20,41464,191,10,10,4,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,10-Jul-2012,Jul 10th 2012,Wed 07-10-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-10,Wed ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-10,2011-07-10,2012-04-10,2012-01-10,2012-06-10,2012-05-10,2012-07-03,2012-06-26,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-27,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-273,0,-3,-9,-39,-39,41331,58,58,27,4,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,27-Feb-2012,Feb 27th 2012,Wed 02-27-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-02-27,Wed ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-02-27,2011-02-27,2012-11-27,2012-08-27,2012-01-27,2012-12-27,2012-02-20,2012-02-13,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-02,2012-01-01,2012-01-01,2012-03-01,2012-02-24,0,-3,-8,-39,-270,0,-3,-9,-39,-39,41334,61,61,2,7,9,5904,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-02,02-Mar-2012,Mar 2nd 2012,Sat 03-02-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-03-02,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-03-02,2011-03-02,2012-12-02,2012-09-02,2012-02-02,2012-01-02,2012-02-23,2012-02-16,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-02,2012-01-01,2012-10-01,2012-11-01,2012-10-27,0,0,0,-4,-25,0,0,-1,-4,-4,41579,306,33,2,7,44,5939,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-02,02-Nov-2012,Nov 2nd 2012,Sat 11-02-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-10-27,2012-10-28,2012-11-02,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,44,2012,2012-10-27,2012-11-02,N,Wk.44 - 13,2012-10-27 00:00:00,2012-11-02 00:00:00,2012W44 ,2012W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2012,N,2012-11-02,2011-11-02,2012-08-02,2012-05-02,2012-10-02,2012-09-02,2012-10-26,2012-10-19,0,0,0,0,0,0,0,0,11,4,44,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-21,2012-01-01,2012-01-01,2012-03-01,2012-03-17,0,-3,-8,-36,-251,0,-3,-9,-36,-36,41353,80,80,21,5,12,5907,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-23,21-Mar-2012,Mar 21st 2012,Thu 03-21-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-17,2012-03-18,2012-03-21,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,12,2012,2012-03-17,2012-03-23,N,Wk.12 - 13,2012-03-17 00:00:00,2012-03-23 00:00:00,2012W12 ,2012W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2012,N,2012-03-21,2011-03-21,2012-12-21,2012-09-21,2012-02-21,2012-01-21,2012-03-14,2012-03-07,0,0,0,0,0,0,0,0,3,1,12,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-03,2012-01-01,2012-04-01,2012-04-01,2012-03-31,0,-2,-7,-34,-238,0,-2,-8,-34,-34,41366,93,3,3,4,14,5909,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-06,03-Apr-2012,Apr 3rd 2012,Wed 04-03-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-03-31,2012-04-01,2012-04-03,Wed ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,14,2012,2012-03-31,2012-04-06,N,Wk.14 - 13,2012-03-31 00:00:00,2012-04-06 00:00:00,2012W14 ,2012W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2012,N,2012-04-03,2011-04-03,2012-01-03,2012-10-03,2012-03-03,2012-02-03,2012-03-27,2012-03-20,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-06,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-144,0,-1,-5,-21,-21,41460,187,6,6,7,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,06-Jul-2012,Jul 6th 2012,Sat 07-06-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-06,Sat ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-06,2011-07-06,2012-04-06,2012-01-06,2012-06-06,2012-05-06,2012-06-29,2012-06-22,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-26,2012-01-01,2012-04-01,2012-06-01,2012-06-23,0,-2,-5,-22,-154,0,-2,-6,-22,-22,41450,177,87,26,4,26,5921,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-29,26-Jun-2012,Jun 26th 2012,Wed 06-26-13,1,0,0,0,2012-06-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-23,2012-06-24,2012-06-26,Wed ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,26,2012,2012-06-23,2012-06-29,N,Wk.26 - 13,2012-06-23 00:00:00,2012-06-29 00:00:00,2012W26 ,2012W26 ,06/23/13 - 06/29/13,06/23 - 06/29,2012,N,2012-06-26,2011-06-26,2012-03-26,2012-12-26,2012-05-26,2012-04-26,2012-06-19,2012-06-12,0,0,0,0,0,0,0,0,6,2,26,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-26,2012-01-01,2012-07-01,2012-08-01,2012-08-25,0,-1,-3,-13,-93,0,-1,-3,-13,-13,41511,238,57,26,2,35,5930,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-31,26-Aug-2012,Aug 26th 2012,Mon 08-26-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-08-25,2012-08-25,2012-08-26,2012-08-26,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,35,2012,2012-08-25,2012-08-31,N,Wk.35 - 13,2012-08-25 00:00:00,2012-08-31 00:00:00,2012W35 ,2012W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2012,N,2012-08-26,2011-08-26,2012-05-26,2012-02-26,2012-07-26,2012-06-26,2012-08-19,2012-08-12,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-13,2012-01-01,2012-10-01,2012-12-01,2012-12-08,0,0,1,2,16,0,0,0,2,2,41620,347,74,13,6,50,5945,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-14,13-Dec-2012,Dec 13th 2012,Fri 12-13-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-08,2012-12-09,2012-12-13,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,50,2012,2012-12-08,2012-12-14,N,Wk.50 - 13,2012-12-08 00:00:00,2012-12-14 00:00:00,2012W50 ,2012W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2012,N,2012-12-13,2011-12-13,2012-09-13,2012-06-13,2012-11-13,2012-10-13,2012-12-06,2012-11-29,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-02,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-148,0,-1,-5,-21,-21,41456,183,2,2,3,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,02-Jul-2012,Jul 2nd 2012,Tue 07-02-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-02,Tue ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-02,2011-07-02,2012-04-02,2012-01-02,2012-06-02,2012-05-02,2012-06-25,2012-06-18,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-05,2012-01-01,2012-07-01,2012-08-01,2012-08-04,0,-1,-3,-16,-114,0,-1,-4,-16,-16,41490,217,36,5,2,32,5927,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-10,05-Aug-2012,Aug 5th 2012,Mon 08-05-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-04,2012-08-05,2012-08-05,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,32,2012,2012-08-04,2012-08-10,N,Wk.32 - 13,2012-08-04 00:00:00,2012-08-10 00:00:00,2012W32 ,2012W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2012,N,2012-08-05,2011-08-05,2012-05-05,2012-02-05,2012-07-05,2012-06-05,2012-07-29,2012-07-22,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-08,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-292,0,-3,-10,-42,-42,41312,39,39,8,6,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,08-Feb-2012,Feb 8th 2012,Fri 02-08-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-08,Fri ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-08,2011-02-08,2012-11-08,2012-08-08,2012-01-08,2012-12-08,2012-02-01,2012-01-25,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-14,2012-01-01,2012-01-01,2012-02-01,2012-02-10,0,-3,-9,-41,-286,0,-3,-10,-41,-41,41318,45,45,14,5,7,5902,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-16,14-Feb-2012,Feb 14th 2012,Thu 02-14-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-10,2012-02-11,2012-02-14,Thu ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,7,2012,2012-02-10,2012-02-16,N,Wk.07 - 13,2012-02-10 00:00:00,2012-02-16 00:00:00,2012W07 ,2012W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2012,N,2012-02-14,2011-02-14,2012-11-14,2012-08-14,2012-01-14,2012-12-14,2012-02-07,2012-01-31,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-21,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-67,0,-1,-3,-10,-10,41537,264,83,21,7,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,21-Sep-2012,Sep 21st 2012,Sat 09-21-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-21,Sat ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-21,2011-09-21,2012-06-21,2012-03-21,2012-08-21,2012-07-21,2012-09-14,2012-09-07,0,0,0,0,0,0,0,0,9,3,38,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-19,2012-01-01,2012-01-01,2012-03-01,2012-03-17,0,-3,-8,-36,-253,0,-3,-9,-36,-36,41351,78,78,19,3,12,5907,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-23,19-Mar-2012,Mar 19th 2012,Tue 03-19-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-17,2012-03-18,2012-03-19,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,12,2012,2012-03-17,2012-03-23,N,Wk.12 - 13,2012-03-17 00:00:00,2012-03-23 00:00:00,2012W12 ,2012W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2012,N,2012-03-19,2011-03-19,2012-12-19,2012-09-19,2012-02-19,2012-01-19,2012-03-12,2012-03-05,0,0,0,0,0,0,0,0,3,1,12,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-28,2012-01-01,2012-01-01,2012-01-01,2012-01-27,0,-3,-10,-43,-303,0,-3,-10,-43,-43,41301,28,28,28,2,5,5900,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-02-02,28-Jan-2012,Jan 28th 2012,Mon 01-28-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-01-27,2012-01-27,2012-01-28,2012-01-28,Mon ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,5,2012,2012-01-27,2012-02-02,N,Wk.05 - 13,2012-01-27 00:00:00,2012-02-02 00:00:00,2012W05 ,2012W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2012,N,2012-01-28,2011-01-28,2012-10-28,2012-07-28,2012-12-28,2012-11-28,2012-01-21,2012-01-14,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-03,2012-01-01,2012-07-01,2012-08-01,2012-07-28,0,-1,-3,-17,-116,0,-1,-4,-17,-17,41488,215,34,3,7,31,5926,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-03,03-Aug-2012,Aug 3rd 2012,Sat 08-03-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-07-28,2012-07-29,2012-08-03,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,31,2012,2012-07-28,2012-08-03,N,Wk.31 - 13,2012-07-28 00:00:00,2012-08-03 00:00:00,2012W31 ,2012W31 ,07/28/13 - 08/03/13,07/28 - 08/03,2012,N,2012-08-03,2011-08-03,2012-05-03,2012-02-03,2012-07-03,2012-06-03,2012-07-27,2012-07-20,0,0,0,0,0,0,0,0,8,3,31,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-14,2012-01-01,2012-04-01,2012-06-01,2012-06-09,0,-2,-5,-24,-166,0,-2,-6,-24,-24,41438,165,75,14,6,24,5919,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-15,14-Jun-2012,Jun 14th 2012,Fri 06-14-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-09,2012-06-10,2012-06-14,Fri ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,24,2012,2012-06-09,2012-06-15,N,Wk.24 - 13,2012-06-09 00:00:00,2012-06-15 00:00:00,2012W24 ,2012W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2012,N,2012-06-14,2011-06-14,2012-03-14,2012-12-14,2012-05-14,2012-04-14,2012-06-07,2012-05-31,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-08,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-142,0,-1,-5,-20,-20,41462,189,8,8,2,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,08-Jul-2012,Jul 8th 2012,Mon 07-08-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-08,Mon ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-08,2011-07-08,2012-04-08,2012-01-08,2012-06-08,2012-05-08,2012-07-01,2012-06-24,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-30,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,3,0,0,0,0,0,41607,334,61,30,7,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,30-Nov-2012,Nov 30th 2012,Sat 11-30-13,0,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-25,2012-11-30,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,N,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-30,2011-11-30,2012-08-30,2012-05-30,2012-10-30,2012-09-30,2012-11-23,2012-11-16,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-10,2012-01-01,2012-01-01,2012-01-01,2012-01-06,0,-3,-10,-46,-321,0,-3,-11,-46,-46,41283,10,10,10,5,2,5897,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-12,10-Jan-2012,Jan 10th 2012,Thu 01-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-06,2012-01-07,2012-01-10,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,2,2012,2012-01-06,2012-01-12,N,Wk.02 - 13,2012-01-06 00:00:00,2012-01-12 00:00:00,2012W02 ,2012W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2012,N,2012-01-10,2011-01-10,2012-10-10,2012-07-10,2012-12-10,2012-11-10,2012-01-03,2012-12-27,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-18,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-193,0,-2,-7,-28,-28,41411,138,48,18,7,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,18-May-2012,May 18th 2012,Sat 05-18-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-18,Sat ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-18,2011-05-18,2012-02-18,2012-11-18,2012-04-18,2012-03-18,2012-05-11,2012-05-04,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-15,2012-01-01,2012-04-01,2012-04-01,2012-04-14,0,-2,-7,-32,-226,0,-2,-8,-32,-32,41378,105,15,15,2,16,5911,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-20,15-Apr-2012,Apr 15th 2012,Mon 04-15-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-14,2012-04-15,2012-04-15,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,16,2012,2012-04-14,2012-04-20,N,Wk.16 - 13,2012-04-14 00:00:00,2012-04-20 00:00:00,2012W16 ,2012W16 ,04/14/13 - 04/20/13,04/14 - 04/20,2012,N,2012-04-15,2011-04-15,2012-01-15,2012-10-15,2012-03-15,2012-02-15,2012-04-08,2012-04-01,0,0,0,0,0,0,0,0,4,2,16,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-12,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-260,0,-3,-9,-37,-37,41344,71,71,12,3,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,12-Mar-2012,Mar 12th 2012,Tue 03-12-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-11,2012-03-12,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,N,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-12,2011-03-12,2012-12-12,2012-09-12,2012-02-12,2012-01-12,2012-03-05,2012-02-26,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-09,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-18,0,0,-1,-3,-3,41586,313,40,9,7,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,09-Nov-2012,Nov 9th 2012,Sat 11-09-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-09,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-09,2011-11-09,2012-08-09,2012-05-09,2012-10-09,2012-09-09,2012-11-02,2012-10-26,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-14,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-258,0,-3,-9,-37,-37,41346,73,73,14,5,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,14-Mar-2012,Mar 14th 2012,Thu 03-14-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-11,2012-03-14,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,N,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-14,2011-03-14,2012-12-14,2012-09-14,2012-02-14,2012-01-14,2012-03-07,2012-02-28,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-03,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-85,0,-1,-3,-12,-12,41519,246,65,3,3,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,03-Sep-2012,Sep 3rd 2012,Tue 09-03-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-09-02,2012-09-03,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,N,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-03,2011-09-03,2012-06-03,2012-03-03,2012-08-03,2012-07-03,2012-08-27,2012-08-20,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-16,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-195,0,-2,-7,-28,-28,41409,136,46,16,5,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,16-May-2012,May 16th 2012,Thu 05-16-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-16,Thu ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-16,2011-05-16,2012-02-16,2012-11-16,2012-04-16,2012-03-16,2012-05-09,2012-05-02,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-27,2012-01-01,2012-10-01,2012-10-01,2012-10-27,0,0,-1,-4,-31,0,0,-1,-4,-5,41573,300,27,27,1,44,5939,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-11-02,27-Oct-2012,Oct 27th 2012,Sun 10-27-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-10-27,2012-10-27,2012-10-21,2012-10-27,Sun ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,44,2012,2012-10-27,2012-11-02,Y,Wk.44 - 13,2012-10-27 00:00:00,2012-11-02 00:00:00,2012W44 ,2012W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2012,N,2012-10-27,2011-10-27,2012-07-27,2012-04-27,2012-09-27,2012-08-27,2012-10-20,2012-10-13,0,0,0,0,1,0,0,0,11,4,44,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-21,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-98,0,-1,-4,-14,-14,41506,233,52,21,4,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,21-Aug-2012,Aug 21st 2012,Wed 08-21-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-21,Wed ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-21,2011-08-21,2012-05-21,2012-02-21,2012-07-21,2012-06-21,2012-08-14,2012-08-07,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-25,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-33,0,0,-2,-5,-5,41571,298,25,25,6,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,25-Oct-2012,Oct 25th 2012,Fri 10-25-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-25,Fri ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-25,2011-10-25,2012-07-25,2012-04-25,2012-09-25,2012-08-25,2012-10-18,2012-10-11,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-24,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-64,0,-1,-3,-9,-9,41540,267,86,24,3,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,24-Sep-2012,Sep 24th 2012,Tue 09-24-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-23,2012-09-24,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,N,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-24,2011-09-24,2012-06-24,2012-03-24,2012-08-24,2012-07-24,2012-09-17,2012-09-10,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-27,2012-01-01,2012-01-01,2012-01-01,2012-01-27,0,-3,-10,-43,-304,0,-3,-10,-43,-44,41300,27,27,27,1,5,5900,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-02-02,27-Jan-2012,Jan 27th 2012,Sun 01-27-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-01-27,2012-01-27,2012-01-21,2012-01-27,Sun ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,5,2012,2012-01-27,2012-02-02,Y,Wk.05 - 13,2012-01-27 00:00:00,2012-02-02 00:00:00,2012W05 ,2012W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2012,N,2012-01-27,2011-01-27,2012-10-27,2012-07-27,2012-12-27,2012-11-27,2012-01-20,2012-01-13,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-10,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-262,0,-3,-9,-37,-38,41342,69,69,10,1,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,10-Mar-2012,Mar 10th 2012,Sun 03-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-04,2012-03-10,Sun ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,Y,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-10,2011-03-10,2012-12-10,2012-09-10,2012-02-10,2012-01-10,2012-03-03,2012-02-24,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-14,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-197,0,-2,-7,-28,-28,41407,134,44,14,3,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,14-May-2012,May 14th 2012,Tue 05-14-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-14,Tue ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-14,2011-05-14,2012-02-14,2012-11-14,2012-04-14,2012-03-14,2012-05-07,2012-04-30,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-31,2012-01-01,2012-10-01,2012-12-01,2012-12-29,0,0,1,5,34,1,1,1,5,5,41638,365,92,31,3,53,5948,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-31,31-Dec-2012,Dec 31st 2012,Tue 12-31-13,0,0,0,0,2012-12-21,365,92,31,3,2012-12-29,2012-12-29,2012-12-29,2012-12-29,2012-12-30,2012-12-31,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,1,2014,2012-12-29,2014-01-04,N,Wk.53 - 13,2012-12-29 00:00:00,2012-12-31 00:00:00,2012W53 ,2012W53 ,12/29/13 - 12/31/13,12/29 - 12/31,2012,N,2012-12-31,2011-12-31,2012-09-30,2012-06-30,2012-11-30,2012-10-31,2012-12-24,2012-12-17,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-01,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-87,0,-1,-3,-12,-13,41517,244,63,1,1,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,01-Sep-2012,Sep 1st 2012,Sun 09-01-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-08-26,2012-09-01,Sun ,2012M09,Sep-2012,Y,2012M09 ,Y,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,Y,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-01,2011-09-01,2012-06-01,2012-03-01,2012-08-01,2012-07-01,2012-08-25,2012-08-18,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-13,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-228,0,-2,-8,-33,-33,41376,103,13,13,7,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,13-Apr-2012,Apr 13th 2012,Sat 04-13-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-13,Sat ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-13,2011-04-13,2012-01-13,2012-10-13,2012-03-13,2012-02-13,2012-04-06,2012-03-30,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-09,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-291,0,-3,-10,-42,-42,41313,40,40,9,7,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,09-Feb-2012,Feb 9th 2012,Sat 02-09-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-09,Sat ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-09,2011-02-09,2012-11-09,2012-08-09,2012-01-09,2012-12-09,2012-02-02,2012-01-26,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-27,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,30,0,0,0,4,4,41634,361,88,27,6,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,27-Dec-2012,Dec 27th 2012,Fri 12-27-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-27,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-27,2011-12-27,2012-09-27,2012-06-27,2012-11-27,2012-10-27,2012-12-20,2012-12-13,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-19,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-100,0,-1,-4,-14,-14,41504,231,50,19,2,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,19-Aug-2012,Aug 19th 2012,Mon 08-19-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-19,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-19,2011-08-19,2012-05-19,2012-02-19,2012-07-19,2012-06-19,2012-08-12,2012-08-05,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-11,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-230,0,-2,-8,-33,-33,41374,101,11,11,5,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,11-Apr-2012,Apr 11th 2012,Thu 04-11-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-11,Thu ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-11,2011-04-11,2012-01-11,2012-10-11,2012-03-11,2012-02-11,2012-04-04,2012-03-28,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-06,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,9,0,0,0,1,1,41613,340,67,6,6,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,06-Dec-2012,Dec 6th 2012,Fri 12-06-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-06,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-06,2011-12-06,2012-09-06,2012-06-06,2012-11-06,2012-10-06,2012-11-29,2012-11-22,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-31,2012-01-01,2012-01-01,2012-03-01,2012-03-31,0,-3,-8,-34,-241,0,-2,-8,-34,-35,41363,90,90,31,1,14,5909,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-04-06,31-Mar-2012,Mar 31st 2012,Sun 03-31-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-03-31,2012-03-31,2012-03-31,2012-03-25,2012-03-31,Sun ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,14,2012,2012-03-31,2012-04-06,Y,Wk.14 - 13,2012-03-31 00:00:00,2012-04-06 00:00:00,2012W14 ,2012W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2012,N,2012-03-31,2011-03-31,2012-12-31,2012-09-30,2012-02-28,2012-01-31,2012-03-24,2012-03-17,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-22,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-66,0,-1,-3,-9,-10,41538,265,84,22,1,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,22-Sep-2012,Sep 22nd 2012,Sun 09-22-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-16,2012-09-22,Sun ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,Y,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-22,2011-09-22,2012-06-22,2012-03-22,2012-08-22,2012-07-22,2012-09-15,2012-09-08,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-06,2012-01-01,2012-01-01,2012-01-01,2012-01-06,0,-3,-10,-46,-325,0,-3,-11,-46,-47,41279,6,6,6,1,2,5897,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-12,06-Jan-2012,Jan 6th 2012,Sun 01-06-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-06,2012-12-31,2012-01-06,Sun ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,2,2012,2012-01-06,2012-01-12,Y,Wk.02 - 13,2012-01-06 00:00:00,2012-01-12 00:00:00,2012W02 ,2012W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2012,N,2012-01-06,2011-01-06,2012-10-06,2012-07-06,2012-12-06,2012-11-06,2012-12-30,2012-12-23,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-04,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-327,0,-3,-11,-47,-47,41277,4,4,4,6,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,04-Jan-2012,Jan 4th 2012,Fri 01-04-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-04,Fri ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-04,2011-01-04,2012-10-04,2012-07-04,2012-12-04,2012-11-04,2012-12-28,2012-12-21,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-16,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-134,0,-1,-5,-19,-19,41470,197,16,16,3,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,16-Jul-2012,Jul 16th 2012,Tue 07-16-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-16,Tue ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-16,2011-07-16,2012-04-16,2012-01-16,2012-06-16,2012-05-16,2012-07-09,2012-07-02,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-03,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-24,0,0,-1,-3,-4,41580,307,34,3,1,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,03-Nov-2012,Nov 3rd 2012,Sun 11-03-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-10-28,2012-11-03,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,Y,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-03,2011-11-03,2012-08-03,2012-05-03,2012-10-03,2012-09-03,2012-10-27,2012-10-20,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-25,2012-01-01,2012-07-01,2012-08-01,2012-08-25,0,-1,-3,-13,-94,0,-1,-3,-13,-14,41510,237,56,25,1,35,5930,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-31,25-Aug-2012,Aug 25th 2012,Sun 08-25-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-08-25,2012-08-25,2012-08-19,2012-08-25,Sun ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,35,2012,2012-08-25,2012-08-31,Y,Wk.35 - 13,2012-08-25 00:00:00,2012-08-31 00:00:00,2012W35 ,2012W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2012,N,2012-08-25,2011-08-25,2012-05-25,2012-02-25,2012-07-25,2012-06-25,2012-08-18,2012-08-11,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-24,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,-3,0,0,0,0,-1,41601,328,55,24,1,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,24-Nov-2012,Nov 24th 2012,Sun 11-24-13,1,1,1,1,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-18,2012-11-24,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,Y,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-24,2011-11-24,2012-08-24,2012-05-24,2012-10-24,2012-09-24,2012-11-17,2012-11-10,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-12,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-199,0,-2,-7,-28,-29,41405,132,42,12,1,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,12-May-2012,May 12th 2012,Sun 05-12-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-06,2012-05-12,Sun ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,Y,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-12,2011-05-12,2012-02-12,2012-11-12,2012-04-12,2012-03-12,2012-05-05,2012-04-28,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-17,2012-01-01,2012-07-01,2012-08-01,2012-08-11,0,-1,-3,-15,-102,0,-1,-4,-15,-15,41502,229,48,17,7,33,5928,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-17,17-Aug-2012,Aug 17th 2012,Sat 08-17-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-11,2012-08-12,2012-08-17,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,33,2012,2012-08-11,2012-08-17,N,Wk.33 - 13,2012-08-11 00:00:00,2012-08-17 00:00:00,2012W33 ,2012W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2012,N,2012-08-17,2011-08-17,2012-05-17,2012-02-17,2012-07-17,2012-06-17,2012-08-10,2012-08-03,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-21,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-37,0,0,-2,-5,-5,41567,294,21,21,2,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,21-Oct-2012,Oct 21st 2012,Mon 10-21-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-21,Mon ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-21,2011-10-21,2012-07-21,2012-04-21,2012-09-21,2012-08-21,2012-10-14,2012-10-07,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-27,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-245,0,-3,-9,-35,-35,41359,86,86,27,4,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,27-Mar-2012,Mar 27th 2012,Wed 03-27-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-27,Wed ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-27,2011-03-27,2012-12-27,2012-09-27,2012-02-27,2012-01-27,2012-03-20,2012-03-13,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-22,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-5,0,0,-1,-1,-1,41599,326,53,22,6,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,22-Nov-2012,Nov 22nd 2012,Fri 11-22-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-22,Fri ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-22,2011-11-22,2012-08-22,2012-05-22,2012-10-22,2012-09-22,2012-11-15,2012-11-08,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-07,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-143,0,-1,-5,-20,-21,41461,188,7,7,1,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,07-Jul-2012,Jul 7th 2012,Sun 07-07-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-01,2012-07-07,Sun ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,Y,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-07,2011-07-07,2012-04-07,2012-01-07,2012-06-07,2012-05-07,2012-06-30,2012-06-23,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-11,2012-01-01,2012-07-01,2012-09-01,2012-09-08,0,-1,-2,-11,-77,0,-1,-3,-11,-11,41527,254,73,11,4,37,5932,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-14,11-Sep-2012,Sep 11th 2012,Wed 09-11-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-08,2012-09-09,2012-09-11,Wed ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,37,2012,2012-09-08,2012-09-14,N,Wk.37 - 13,2012-09-08 00:00:00,2012-09-14 00:00:00,2012W37 ,2012W37 ,09/08/13 - 09/14/13,09/08 - 09/14,2012,N,2012-09-11,2011-09-11,2012-06-11,2012-03-11,2012-08-11,2012-07-11,2012-09-04,2012-08-28,0,0,0,0,0,0,0,0,9,3,37,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-19,2012-01-01,2012-10-01,2012-10-01,2012-10-13,0,0,-1,-6,-39,0,0,-2,-6,-6,41565,292,19,19,7,42,5937,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-19,19-Oct-2012,Oct 19th 2012,Sat 10-19-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-13,2012-10-14,2012-10-19,Sat ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,42,2012,2012-10-13,2012-10-19,N,Wk.42 - 13,2012-10-13 00:00:00,2012-10-19 00:00:00,2012W42 ,2012W42 ,10/13/13 - 10/19/13,10/13 - 10/19,2012,N,2012-10-19,2011-10-19,2012-07-19,2012-04-19,2012-09-19,2012-08-19,2012-10-12,2012-10-05,0,0,0,0,0,0,0,0,10,4,42,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-23,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,26,0,0,0,4,4,41630,357,84,23,2,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,23-Dec-2012,Dec 23rd 2012,Mon 12-23-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-23,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-23,2011-12-23,2012-09-23,2012-06-23,2012-11-23,2012-10-23,2012-12-16,2012-12-09,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-24,2012-01-01,2012-04-01,2012-05-01,2012-05-19,0,-2,-6,-27,-187,0,-2,-7,-27,-27,41417,144,54,24,6,21,5916,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-25,24-May-2012,May 24th 2012,Fri 05-24-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-19,2012-05-20,2012-05-24,Fri ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,21,2012,2012-05-19,2012-05-25,N,Wk.21 - 13,2012-05-19 00:00:00,2012-05-25 00:00:00,2012W21 ,2012W21 ,05/19/13 - 05/25/13,05/19 - 05/25,2012,N,2012-05-24,2011-05-24,2012-02-24,2012-11-24,2012-04-24,2012-03-24,2012-05-17,2012-05-10,0,0,0,0,0,0,0,0,5,2,21,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-05,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-22,0,0,-1,-3,-3,41582,309,36,5,3,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,05-Nov-2012,Nov 5th 2012,Tue 11-05-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-05,Tue ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-05,2011-11-05,2012-08-05,2012-05-05,2012-10-05,2012-09-05,2012-10-29,2012-10-22,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-29,2012-01-01,2012-04-01,2012-05-01,2012-05-26,0,-2,-6,-26,-182,0,-2,-6,-26,-26,41422,149,59,29,4,22,5917,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-06-01,29-May-2012,May 29th 2012,Wed 05-29-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-05-26,2012-05-26,2012-05-27,2012-05-29,Wed ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,22,2012,2012-05-26,2012-06-01,N,Wk.22 - 13,2012-05-26 00:00:00,2012-06-01 00:00:00,2012W22 ,2012W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2012,N,2012-05-29,2011-05-29,2012-02-28,2012-11-29,2012-04-29,2012-03-29,2012-05-22,2012-05-15,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-02,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-329,0,-3,-11,-47,-47,41275,2,2,2,4,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,02-Jan-2012,Jan 2nd 2012,Wed 01-02-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-02,Wed ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-02,2011-01-02,2012-10-02,2012-07-02,2012-12-02,2012-11-02,2012-12-26,2012-12-19,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-02,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,5,0,0,0,1,1,41609,336,63,2,2,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,02-Dec-2012,Dec 2nd 2012,Mon 12-02-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-02,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-02,2011-12-02,2012-09-02,2012-06-02,2012-11-02,2012-10-02,2012-11-25,2012-11-18,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-24,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-276,0,-3,-9,-39,-40,41328,55,55,24,1,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,24-Feb-2012,Feb 24th 2012,Sun 02-24-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-18,2012-02-24,Sun ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,Y,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-02-24,2011-02-24,2012-11-24,
<TRUNCATED>
[48/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/bin/check-env.sh b/bin/check-env.sh
deleted file mode 100755
index d3cd709..0000000
--- a/bin/check-env.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#by source
-
-if [ -z "$KYLIN_HOME" ]
-then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
-else
- echo "KYLIN_HOME is set to ${KYLIN_HOME}"
-fi
-
-if [ -z "$(command -v hbase version)" ]
-then
- echo "Please make sure the user has the privilege to run hbase shell"
- exit 1
-fi
-
-if [ -z "$(command -v hive --version)" ]
-then
- echo "Please make sure the user has the privilege to run hive shell"
- exit 1
-fi
-
-if [ -z "$(command -v hadoop version)" ]
-then
- echo "Please make sure the user has the privilege to run hadoop shell"
- exit 1
-fi
-
-WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir`
-hadoop fs -mkdir -p $WORKING_DIR
-
-if [ $? != 0 ]
-then
- echo "failed to create $WORKING_DIR, Please make sure the user has right to access $WORKING_DIR"
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/find-hbase-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hbase-dependency.sh b/bin/find-hbase-dependency.sh
deleted file mode 100755
index 03ab2a1..0000000
--- a/bin/find-hbase-dependency.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-hbase_classpath=`hbase classpath`
-arr=(`echo $hbase_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
-hbase_common_path=
-for data in ${arr[@]}
-do
- result=`echo $data | grep -e 'hbase-common[a-z0-9A-Z\.-]*jar' | grep -v tests`
- if [ $result ]
- then
- hbase_common_path=$data
- fi
-done
-
-if [ -z "$hbase_common_path" ]
-then
- echo "hbase-common lib not found"
- exit 1
-fi
-
-hbase_dependency=${hbase_common_path}
-echo "hbase dependency: $hbase_dependency"
-export hbase_dependency
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
deleted file mode 100755
index 5994dda..0000000
--- a/bin/find-hive-dependency.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-hive_env=`hive -e set | grep 'env:CLASSPATH'`
-
-hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
-arr=(`echo $hive_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
-hive_conf_path=
-hive_exec_path=
-for data in ${arr[@]}
-do
- result=`echo $data | grep -e 'hive-exec[a-z0-9A-Z\.-]*jar'`
- if [ $result ]
- then
- hive_exec_path=$data
- fi
- result=`echo $data | grep -e 'hive[^/]*/conf'`
- if [ $result ]
- then
- hive_conf_path=$data
- fi
-done
-
-# in some versions of hive hcatalog is not in hive's classpath, find it separately
-if [ -z "$HCAT_HOME" ]
-then
- echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
- hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
- if [ -d "${hadoop_home}/hive-hcatalog" ]; then
- hcatalog_home=${hadoop_home}/hive-hcatalog
- elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
- hcatalog_home=${hadoop_home}/hive/hcatalog
- else
- echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
- exit 1
- fi
-else
- echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
- hcatalog_home=${HCAT_HOME}
-fi
-
-hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
-
-if [ -z "$hcatalog" ]
-then
- echo "hcatalog lib not found"
- exit 1
-fi
-
-
-hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
-hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
-echo "hive dependency: $hive_dependency"
-export hive_dependency
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/bin/get-properties.sh b/bin/get-properties.sh
deleted file mode 100755
index 4c93b3b..0000000
--- a/bin/get-properties.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ $# != 1 ]
-then
- echo 'invalid input'
- exit -1
-fi
-
-result=
-for i in `cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "$1" | grep -v '^#' |awk -F '=' '{print $2}' | cut -c 1-`
-do
- :
- result=$i
-done
-echo $result
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/health-check.sh
----------------------------------------------------------------------
diff --git a/bin/health-check.sh b/bin/health-check.sh
deleted file mode 100755
index 3e18631..0000000
--- a/bin/health-check.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ALERT="your@email.com"
-
-OUTPUT=$(
- curl --max-time 20 -# \
- --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
- -H "Authorization:Basic QURNSU46S1lMSU4=" \
- -H "Content-Type:application/json;charset=UTF-8" \
- http://localhost:7070/kylin/api/query \
-)
-
-# ----------------------------------------------------------------------------
-
-date
-
-if [[ $OUTPUT == *"results"* ]]; then
- echo "Good."
-else
- echo "Bad."
- TS_FILE=/tmp/kylin_healthmon_ts
- LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
- CURR_TS=`date +%s`
- echo last: $LAST_TS
- echo curr: $CURR_TS
- if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
- echo "Sending mail..."
- echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
- if [ "$?" == "0" ]; then
- touch $TS_FILE
- fi
- fi
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
deleted file mode 100755
index 6b0a495..0000000
--- a/bin/kylin.sh
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-mkdir -p ${KYLIN_HOME}/logs
-
-# start command
-if [ "$1" == "start" ]
-then
-
- if [ -f "${KYLIN_HOME}/pid" ]
- then
- PID=`cat $KYLIN_HOME/pid`
- if ps -p $PID > /dev/null
- then
- echo "Kylin is running, stop it first"
- exit 1
- fi
- fi
-
- tomcat_root=${dir}/../tomcat
- export tomcat_root
-
-
- #The location of all hadoop/hbase configurations are difficult to get.
- #Plus, some of the system properties are secretly set in hadoop/hbase shell command.
- #For example, in hdp 2.2, there is a system property called hdp.version,
- #which we cannot get until running hbase or hadoop shell command.
- #
- #To save all these troubles, we use hbase runjar to start tomcat.
- #In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
- #hbase command will do all the dirty tasks for us:
-
-
-
- useSandbox=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.sandbox`
- spring_profile="default"
- if [ "$useSandbox" = "true" ]
- then spring_profile="sandbox"
- fi
-
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
- fi
-
- export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
- export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-
- #debug if encounter NoClassDefError
- #hbase classpath
-
- hbase ${KYLIN_EXTRA_START_OPTS} \
- -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
- -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
- -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
- -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
- -Djava.endorsed.dirs=${tomcat_root}/endorsed \
- -Dcatalina.base=${tomcat_root} \
- -Dcatalina.home=${tomcat_root} \
- -Djava.io.tmpdir=${tomcat_root}/temp \
- -Dkylin.hive.dependency=${hive_dependency} \
- -Dkylin.hbase.dependency=${hbase_dependency} \
- -Dspring.profiles.active=${spring_profile} \
- org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
- echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
- if [ "$useSandbox" = "true" ]
- then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
- else
- echo "Please visit http://<ip>:7070/kylin"
- fi
- echo "You can check the log at ${tomcat_root}/logs/kylin.log"
- exit 0
-
-# stop command
-elif [ "$1" == "stop" ]
-then
- if [ -f "${KYLIN_HOME}/pid" ]
- then
- PID=`cat $KYLIN_HOME/pid`
- if ps -p $PID > /dev/null
- then
- echo "stopping Kylin:$PID"
- kill $PID
- rm ${KYLIN_HOME}/pid
- exit 0
- else
- echo "Kylin is not running, please check"
- exit 1
- fi
-
- else
- echo "Kylin is not running, please check"
- exit 1
- fi
-
-# tool command
-elif [[ "$1" = org.apache.kylin.* ]]
-then
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv-tool.sh" ]
- then source ${dir}/setenv-tool.sh
- fi
-
- export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*:$hive_dependency:${HBASE_CLASSPATH}
-
- exec hbase -Dlog4j.configuration=kylin-log4j.properties "$@"
-
-else
- echo "usage: kylin.sh start or kylin.sh stop"
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
deleted file mode 100755
index 5d4ef40..0000000
--- a/bin/metastore.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script is for production metadata store manipulation
-# It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
-#
-# If you're a developper of Kylin and want to download sandbox's metadata into your dev machine,
-# take a look at SandboxMetastoreCLI
-
-
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-
-if [ "$1" == "backup" ]
-then
-
- mkdir -p ${KYLIN_HOME}/meta_backups
-
- _now=$(date +"%Y_%m_%d_%H_%M_%S")
- _file="${KYLIN_HOME}/meta_backups/meta_${_now}"
- echo "Starting backup to ${_file}"
- mkdir -p ${_file}
-
- ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool download ${_file}
- echo "metadata store backed up to ${_file}"
-
-elif [ "$1" == "restore" ]
-then
-
- _file=$2
- echo "Starting restoring $_file"
- ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool upload $_file
-
-elif [ "$1" == "reset" ]
-then
-
- ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool reset
-
-elif [ "$1" == "clean" ]
-then
-
- ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.job.hadoop.cube.MetadataCleanupJob "${@:2}"
-
-else
- echo "usage: metastore.sh backup"
- echo " metastore.sh reset"
- echo " metastore.sh restore PATH_TO_LOCAL_META"
- echo " metastore.sh clean [--delete true]"
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/performance-monitor.sh
----------------------------------------------------------------------
diff --git a/bin/performance-monitor.sh b/bin/performance-monitor.sh
deleted file mode 100755
index d8421e3..0000000
--- a/bin/performance-monitor.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
- if [ -z "$KYLIN_HOME" ]
- then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
- else
- echo "KYLIN_HOME is set to ${KYLIN_HOME}"
- fi
-
- dir=$(dirname ${0})
-
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
-
- export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}:$KYLIN_HOME/conf
- _monitorjar=`ls ${KYLIN_HOME}/lib |grep kylin-monitor`
-
- hbase \
- org.apache.hadoop.util.RunJar $KYLIN_HOME/lib/${_monitorjar} org.apache.kylin.monitor.Client
- exit 0
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/sample.sh
----------------------------------------------------------------------
diff --git a/bin/sample.sh b/bin/sample.sh
deleted file mode 100755
index e02ef51..0000000
--- a/bin/sample.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
-echo "Going to create sample tables in hive..."
-cd ${KYLIN_HOME}/sample_cube/data
-hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
-
-echo "Sample hive tables are created successfully; Going to create sample cube..."
-cd ${KYLIN_HOME}
-hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
-echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/setenv.sh b/bin/setenv.sh
deleted file mode 100755
index f2e14cc..0000000
--- a/bin/setenv.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# uncomment following to for it to take effect
-export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC"
-
-# uncomment following to for it to take effect
-# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:${KYLIN_HOME}/lib/CustomAgent.jar -Dcom.ebay.lnp.rmiagent.port=8080"
-
-# uncomment following to for it to take effect
-# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
-
-export KYLIN_EXTRA_START_OPTS=""
-
-if [ ! -z "${KYLIN_JVM_SETTINGS}" ]
-then
- echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings"
-fi
-
-if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
-then
- echo "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- echo "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
-fi
-
-if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
-then
- echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- echo "KYLIN_LD_LIBRARY_SETTINGS is not set, lzo compression at MR and hbase might not work"
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/common/.settings/org.eclipse.core.resources.prefs b/common/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 365bbd6..0000000
--- a/common/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/common/.settings/org.eclipse.jdt.core.prefs b/common/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a903301..0000000
--- a/common/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/common/.settings/org.eclipse.jdt.ui.prefs b/common/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index dece0e6..0000000
--- a/common/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
deleted file mode 100644
index 2f01bde..0000000
--- a/common/pom.xml
+++ /dev/null
@@ -1,137 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-common</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:Common</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <properties>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- </dependency>
- <dependency>
- <groupId>com.ning</groupId>
- <artifactId>compress-lzf</artifactId>
- </dependency>
-
- <!-- Env & Test -->
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-email</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <!-- protobuf version conflict with hbase -->
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase-hadoop2.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>0.9.9-RC1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- <version>1.2</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>${hive-hcatalog.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
[26/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
deleted file mode 100644
index 13c7d85..0000000
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ /dev/null
@@ -1,10000 +0,0 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,10000349
-0,2012-08-28,Others,175750,0,13,23.8563,0,10000927
-0,2012-02-16,ABIN,148324,15,13,88.3418,0,10000005
-0,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,10000209
-0,2012-10-22,Others,140746,100,11,83.4540,0,10000154
-0,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,10000372
-0,2013-04-04,Others,963,0,13,88.5907,0,10000648
-0,2012-04-11,Others,15687,0,15,88.1940,0,10000866
-0,2013-01-19,ABIN,60606,3,13,77.9727,0,10000936
-0,2012-04-30,FP-non GTC,106246,0,14,52.2950,0,10000047
-0,2013-02-03,Auction,45333,0,16,56.3584,0,10000470
-0,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,10000203
-0,2013-07-30,Others,24760,0,16,25.0770,0,10000382
-0,2012-04-17,Auction,31519,0,16,72.7384,0,10000069
-0,2013-04-30,FP-GTC,51582,0,15,75.8200,0,10000085
-0,2013-03-23,FP-GTC,16509,0,15,8.6653,0,10000559
-0,2013-04-01,ABIN,175750,3,16,64.2802,0,10000507
-0,2013-07-12,ABIN,50677,0,-99,24.5987,0,10000938
-0,2012-04-23,FP-GTC,45333,0,5,72.6553,0,10000863
-0,2012-12-19,ABIN,3838,0,12,87.4142,0,10000612
-0,2012-08-15,FP-GTC,62179,0,16,67.4238,0,10000839
-0,2013-08-22,Auction,1357,0,5,18.3600,0,10000303
-0,2013-08-23,ABIN,31519,0,5,67.6344,0,10000877
-0,2013-09-12,FP-GTC,223,0,14,71.7898,0,10000848
-0,2012-03-14,FP-GTC,132939,0,15,26.1496,0,10000815
-0,2013-07-13,Others,67703,3,16,83.9661,0,10000858
-0,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,10000387
-0,2012-08-30,Auction,62179,0,11,21.0496,0,10000393
-0,2013-01-10,FP-GTC,48027,0,13,23.4142,0,10000757
-0,2013-01-25,ABIN,145970,0,5,26.7842,0,10000402
-0,2012-07-05,Auction,48027,0,-99,46.7320,0,10000972
-0,2012-11-07,FP-GTC,20213,0,-99,19.7270,0,10000873
-0,2013-06-20,ABIN,100847,0,5,76.2783,0,10000409
-0,2012-10-08,ABIN,26262,0,5,48.7447,0,10000029
-0,2013-04-30,ABIN,87118,0,15,63.2187,0,10000529
-0,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,10000329
-0,2012-05-13,Others,164261,0,13,84.1246,0,10000688
-0,2013-08-02,ABIN,31519,0,5,37.1504,0,10000905
-0,2012-04-03,Auction,164261,0,11,92.0974,0,10000341
-0,2012-09-02,FP-GTC,31519,0,14,87.7829,0,10000460
-0,2013-10-05,ABIN,13987,0,16,74.0719,0,10000381
-0,2012-01-13,FP-GTC,48027,0,15,71.2637,0,10000915
-0,2013-03-01,Others,13836,0,14,16.7288,0,10000953
-0,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,10000277
-0,2012-02-07,Auction,45333,0,5,64.9770,0,10000040
-0,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,10000500
-0,2013-10-13,Auction,31519,0,5,79.3053,0,10000816
-0,2012-06-11,ABIN,4943,0,5,11.6942,0,10000696
-0,2012-10-18,ABIN,80053,0,-99,54.0933,0,10000604
-0,2012-11-03,Auction,20886,0,5,9.8258,0,10000764
-0,2012-01-15,Auction,44079,0,14,13.0371,0,10000343
-0,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,10000724
-0,2012-02-16,FP-GTC,314,0,13,49.1825,0,10000334
-0,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,10000603
-0,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,10000248
-0,2012-07-24,FP-GTC,150265,15,16,57.5645,0,10000653
-0,2012-10-06,Auction,145970,0,5,44.3091,0,10000082
-0,2013-06-14,ABIN,80053,0,5,20.2388,0,10000632
-0,2013-10-10,FP-non GTC,15868,0,11,28.6290,0,10000523
-0,2012-09-30,ABIN,31673,0,12,75.8658,0,10000719
-0,2012-04-02,FP-GTC,63861,0,5,25.8748,0,10000311
-0,2012-03-16,Others,314,0,-99,9.2733,0,10000409
-0,2012-09-30,FP-non GTC,60340,0,14,74.6610,0,10000790
-0,2012-03-20,Others,87118,0,5,79.3781,0,10000384
-0,2012-09-12,FP-GTC,1120,3,5,61.4408,0,10000645
-0,2013-12-10,FP-non GTC,20886,0,11,14.8140,0,10000943
-0,2012-12-05,FP-GTC,106246,0,12,24.9380,0,10000858
-0,2012-08-21,Auction,43479,0,12,22.2008,0,10000310
-0,2012-11-30,FP-GTC,175750,0,15,35.1161,0,10000440
-0,2013-10-10,Others,95672,0,11,82.3718,0,10000128
-0,2012-10-17,Auction,80053,0,15,9.8166,0,10000105
-0,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,10000439
-0,2012-11-14,ABIN,65,0,11,17.5513,0,10000855
-0,2013-06-24,FP-GTC,16509,0,15,53.9195,0,10000130
-0,2012-08-13,FP-non GTC,65,0,5,69.8447,0,10000548
-0,2012-01-03,Others,38238,0,11,90.3722,0,10000739
-0,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,10000291
-0,2013-11-08,Auction,106246,0,5,38.6729,0,10000310
-0,2013-04-07,Others,73506,0,-99,69.8999,0,10000235
-0,2012-10-15,FP-GTC,314,0,13,44.1598,0,10000265
-0,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,10000704
-0,2012-08-17,ABIN,32876,0,16,21.2934,0,10000163
-0,2013-07-09,Auction,37831,0,15,38.2213,0,10000557
-0,2013-09-21,Others,45333,0,13,25.5384,0,10000184
-0,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,10000940
-0,2013-04-05,Auction,11554,0,14,41.8676,0,10000384
-0,2013-03-25,FP-GTC,43398,0,14,75.0110,0,10000697
-0,2012-05-03,ABIN,33038,15,12,64.6735,0,10000147
-0,2012-10-13,ABIN,95672,3,14,3.2128,0,10000547
-0,2012-09-11,FP-GTC,37831,0,16,60.3228,0,10000899
-0,2012-08-31,Auction,314,0,5,13.2480,0,10000350
-0,2012-12-25,Others,165888,0,16,60.3492,0,10000367
-0,2012-12-26,FP-non GTC,50677,0,5,58.6078,0,10000523
-0,2012-05-24,FP-GTC,145970,0,5,33.1342,0,10000926
-0,2012-06-27,Auction,174106,3,13,55.3039,0,10000430
-0,2012-01-02,ABIN,87118,0,14,48.2433,0,10000731
-0,2013-12-14,FP-GTC,50508,0,5,51.5822,0,10000977
-0,2012-12-30,Auction,63889,0,12,3.8393,0,10000957
-0,2013-10-26,FP-non GTC,24541,0,5,23.6812,0,10000350
-0,2013-12-29,ABIN,38238,0,15,99.4389,0,10000605
-0,2012-05-26,FP-GTC,100847,0,15,11.0672,0,10000291
-0,2012-12-18,FP-non GTC,164261,0,11,54.9352,0,10000509
-0,2013-03-12,ABIN,95672,0,15,85.7152,0,10000742
-0,2013-02-22,ABIN,43398,0,11,65.1970,0,10000294
-0,2012-02-20,Others,175750,0,14,63.2187,0,10000364
-0,2013-08-06,Others,32876,0,13,50.8111,0,10000542
-0,2013-06-27,Others,13987,0,13,2.3165,0,10000979
-0,2012-07-17,Others,13987,0,16,25.2136,0,10000744
-0,2013-10-12,FP-non GTC,16509,0,5,15.3986,0,10000652
-0,2012-12-07,Others,158798,0,5,1.6111,0,10000149
-0,2013-02-09,Auction,32876,0,5,37.3237,0,10000467
-0,2012-10-26,Others,44079,0,12,18.1870,0,10000420
-0,2012-04-10,ABIN,132939,0,13,88.5950,0,10000103
-0,2013-04-08,FP-non GTC,159184,0,5,92.4421,0,10000933
-0,2012-05-30,Others,80287,0,14,68.1663,0,10000195
-0,2012-02-23,FP-GTC,88750,0,11,19.4075,0,10000038
-0,2013-10-25,ABIN,11554,0,11,27.2512,0,10000228
-0,2012-12-02,FP-GTC,41940,0,14,49.5504,0,10000927
-0,2013-09-16,FP-GTC,314,0,-99,3.4785,0,10000304
-0,2013-11-01,Others,31519,3,15,46.1783,0,10000771
-0,2012-10-21,FP-non GTC,31519,0,15,1.7696,0,10000507
-0,2013-08-25,Auction,95173,0,13,89.2083,0,10000868
-0,2013-11-09,FP-non GTC,150047,3,12,58.2634,0,10000497
-0,2012-02-04,FP-GTC,175750,0,11,41.3850,0,10000947
-0,2012-10-17,ABIN,26249,0,14,10.5789,0,10000965
-0,2012-03-16,FP-non GTC,963,0,5,17.8949,0,10000908
-0,2012-08-06,Others,94847,0,13,96.0332,0,10000595
-0,2013-12-23,FP-non GTC,24760,0,16,90.7096,0,10000700
-0,2013-03-29,FP-GTC,9426,3,13,50.9023,0,10000090
-0,2013-03-25,Others,20886,0,-99,35.0414,0,10000171
-0,2013-12-10,Auction,95672,0,15,29.8872,0,10000216
-0,2012-03-14,Others,12688,0,14,10.4067,0,10000626
-0,2013-11-29,FP-non GTC,31519,0,13,89.2334,0,10000199
-0,2013-04-29,ABIN,31519,0,-99,99.2460,0,10000565
-0,2012-02-01,FP-GTC,161567,15,11,56.3989,0,10000651
-0,2012-03-17,FP-GTC,4943,0,16,0.4422,0,10000829
-0,2012-11-06,FP-GTC,156614,0,12,2.3454,0,10000370
-0,2012-12-03,Auction,155226,0,13,51.1354,0,10000867
-0,2012-09-09,FP-GTC,99985,0,12,9.1236,0,10000281
-0,2012-06-16,ABIN,99985,0,16,16.4070,0,10000143
-0,2012-05-06,FP-GTC,99985,0,14,6.4304,0,10000514
-0,2013-10-20,FP-GTC,36250,0,13,55.6895,0,10000390
-0,2012-10-11,ABIN,106246,0,15,30.6373,0,10000980
-0,2013-08-08,Others,103178,15,12,41.4904,0,10000751
-0,2012-09-05,Auction,314,0,16,3.2615,0,10000280
-0,2012-08-07,ABIN,48904,0,13,57.7688,0,10000245
-0,2013-07-05,ABIN,314,0,14,28.9444,0,10000888
-0,2012-09-10,FP-non GTC,95672,0,-99,54.0914,0,10000601
-0,2012-09-18,Auction,25147,0,12,74.4422,0,10000679
-0,2012-08-24,FP-non GTC,175750,3,16,45.8282,0,10000397
-0,2013-05-29,Others,26249,0,14,12.7681,0,10000967
-0,2012-11-17,FP-GTC,94847,0,14,74.4181,0,10000175
-0,2012-09-22,ABIN,175750,0,15,85.6879,0,10000453
-0,2013-09-17,ABIN,533,0,11,85.5838,0,10000642
-0,2013-03-28,Others,60340,0,12,72.6495,0,10000397
-0,2013-08-09,Others,216,0,15,88.6918,0,10000867
-0,2013-12-19,Others,43398,0,-99,88.0444,0,10000482
-0,2013-12-02,FP-non GTC,15115,0,-99,29.6027,0,10000538
-0,2012-11-23,Others,46575,0,5,49.3124,0,10000678
-0,2013-06-25,FP-non GTC,16145,0,15,7.4413,0,10000897
-0,2012-01-22,FP-GTC,164,0,14,71.8071,0,10000510
-0,2013-04-24,FP-GTC,1504,0,12,95.5706,0,10000554
-0,2012-03-05,FP-non GTC,12688,0,5,47.7646,0,10000175
-0,2013-02-15,FP-non GTC,16145,0,13,40.9511,0,10000647
-0,2013-01-18,Others,36250,0,13,77.3572,0,10000687
-0,2012-06-04,Others,15115,0,15,25.3400,0,10000493
-0,2013-10-13,FP-GTC,165888,0,-99,58.1861,0,10000753
-0,2012-05-21,Auction,145970,0,5,37.2504,0,10000292
-0,2012-04-26,Auction,11848,0,13,29.2954,0,10000804
-0,2013-09-26,FP-GTC,61323,0,13,84.7095,0,10000761
-0,2013-03-06,Auction,100847,0,-99,81.7980,0,10000593
-0,2012-08-08,ABIN,51582,0,15,15.9790,0,10000138
-0,2012-01-09,Others,1504,0,14,34.0508,0,10000954
-0,2013-10-20,ABIN,20886,0,12,36.9353,0,10000408
-0,2013-06-25,Auction,53064,0,11,99.7714,0,10000409
-0,2013-05-03,FP-GTC,63861,0,5,39.2974,0,10000668
-0,2013-01-26,FP-GTC,100847,0,13,59.5009,0,10000536
-0,2012-09-19,FP-GTC,164261,0,13,15.7583,0,10000231
-0,2013-11-16,FP-GTC,164261,0,16,83.6234,0,10000927
-0,2013-05-07,FP-non GTC,20485,0,5,52.9100,0,10000491
-0,2013-08-15,Others,32996,15,12,54.7122,0,10000411
-0,2013-10-31,FP-GTC,63861,0,14,0.3599,0,10000903
-0,2012-06-13,FP-non GTC,159184,0,11,49.9550,0,10000849
-0,2012-12-25,Auction,20213,0,-99,11.4752,0,10000583
-0,2013-12-15,Auction,759,0,13,33.6991,0,10000390
-0,2012-04-30,Others,87118,0,16,51.4630,0,10000572
-0,2012-07-14,Others,10866,0,11,36.9614,0,10000467
-0,2013-06-25,FP-non GTC,1357,0,12,17.5618,0,10000426
-0,2013-02-06,Auction,13836,0,12,74.3179,0,10000346
-0,2012-06-03,ABIN,60340,0,12,82.3722,0,10000531
-0,2013-07-23,FP-GTC,139973,0,-99,75.2548,0,10000385
-0,2013-07-06,Others,11554,0,14,40.1129,0,10000724
-0,2013-08-10,FP-non GTC,31519,0,-99,5.2432,0,10000356
-0,2013-07-31,Auction,31673,0,14,51.1089,0,10000598
-0,2013-12-05,Auction,314,0,13,19.9031,0,10000068
-0,2012-12-30,ABIN,50508,0,5,73.8659,0,10000482
-0,2012-09-22,Auction,165888,0,12,81.9826,0,10000832
-0,2012-07-16,Auction,223,0,5,76.0284,0,10000354
-0,2013-01-26,Others,20485,0,12,71.6623,0,10000552
-0,2012-11-07,FP-non GTC,57013,0,16,73.2931,0,10000801
-0,2012-01-26,FP-non GTC,175750,0,13,18.8970,0,10000649
-0,2012-06-09,FP-non GTC,60340,0,12,11.1614,0,10000830
-0,2012-06-20,Auction,57990,3,5,69.1183,0,10000557
-0,2013-01-01,FP-non GTC,45333,0,13,94.0586,0,10000253
-0,2013-07-26,FP-non GTC,139973,0,11,39.7239,0,10000441
-0,2012-09-15,FP-GTC,24541,0,15,0.3239,0,10000379
-0,2013-09-22,Auction,31519,3,12,80.0315,0,10000436
-0,2012-10-27,FP-GTC,223,0,15,39.3790,0,10000420
-0,2012-11-20,FP-non GTC,533,0,5,49.4951,0,10000390
-0,2013-12-07,FP-GTC,1349,0,15,93.3008,0,10000398
-0,2012-05-30,FP-non GTC,46575,0,12,66.7652,0,10000915
-0,2012-09-07,FP-non GTC,95672,0,14,29.5970,0,10000231
-0,2013-04-26,Auction,31673,0,-99,15.8347,0,10000260
-0,2012-09-11,FP-non GTC,50508,0,14,80.5444,0,10000263
-0,2013-09-03,FP-non GTC,95173,0,16,33.1487,0,10000493
-0,2012-10-03,FP-non GTC,44079,0,15,99.7408,0,10000828
-0,2012-12-26,ABIN,16145,0,14,40.7826,0,10000062
-0,2012-07-17,Auction,26249,0,15,28.6663,0,10000534
-0,2013-06-30,ABIN,63861,0,14,75.6832,0,10000944
-0,2012-01-09,ABIN,156614,0,11,98.4996,0,10000353
-0,2013-09-08,FP-GTC,156614,0,13,12.7473,0,10000063
-0,2013-09-23,Auction,41940,0,5,37.5254,0,10000383
-0,2012-11-11,Auction,145970,0,5,77.1361,0,10000420
-0,2013-08-26,Others,11848,0,16,28.2799,0,10000323
-0,2012-09-15,Others,11554,0,13,63.7232,0,10000729
-0,2012-01-09,FP-non GTC,25147,0,11,7.3009,0,10000745
-0,2012-08-16,Others,87118,0,15,47.8997,0,10000240
-0,2013-12-26,FP-non GTC,16145,0,12,51.0442,0,10000050
-0,2012-03-24,ABIN,48027,0,5,26.2083,0,10000600
-0,2012-08-22,ABIN,216,0,5,79.9268,0,10000706
-0,2013-11-03,FP-GTC,24541,0,12,83.0235,0,10000293
-0,2013-05-18,FP-non GTC,88750,0,-99,84.8856,0,10000147
-0,2013-01-15,ABIN,95672,0,15,86.0386,0,10000590
-0,2012-12-03,ABIN,50508,0,11,67.9981,0,10000582
-0,2012-06-19,FP-GTC,148324,15,15,53.4811,0,10000793
-0,2013-10-06,Auction,6762,0,14,73.5338,0,10000544
-0,2013-12-18,ABIN,63861,0,5,50.2892,0,10000769
-0,2013-12-04,Auction,25147,0,5,48.3303,0,10000590
-0,2012-04-25,FP-non GTC,62179,0,14,72.3156,0,10000102
-0,2013-11-20,Auction,24760,0,14,91.4883,0,10000370
-0,2013-02-08,Others,20485,0,11,29.1083,0,10000261
-0,2013-04-18,Others,2023,0,5,48.4321,0,10000355
-0,2012-11-21,Others,32876,0,12,16.7365,0,10000352
-0,2012-03-07,FP-GTC,61323,0,-99,32.5836,0,10000494
-0,2012-01-20,FP-non GTC,175750,0,16,52.8153,0,10000025
-0,2013-10-06,FP-GTC,26249,0,-99,1.9539,0,10000660
-0,2012-01-10,Auction,57990,0,14,57.5800,0,10000830
-0,2013-09-13,Others,46575,0,15,60.3315,0,10000528
-0,2012-11-19,FP-non GTC,44079,0,11,73.0457,0,10000482
-0,2012-09-23,Auction,95672,0,13,84.6345,0,10000586
-0,2013-08-31,Others,216,0,12,34.7429,0,10000676
-0,2013-04-11,FP-GTC,963,0,-99,77.7226,0,10000367
-0,2013-03-28,ABIN,15687,0,-99,8.2384,0,10000542
-0,2012-05-24,ABIN,152801,0,15,39.0204,0,10000277
-0,2013-03-05,ABIN,67698,0,16,40.6091,0,10000886
-0,2013-08-06,Auction,13987,0,13,36.5088,0,10000628
-0,2013-02-04,ABIN,20213,0,11,73.5301,0,10000134
-0,2012-05-19,FP-non GTC,38238,0,12,59.3237,0,10000500
-0,2013-08-19,ABIN,57990,0,12,10.1688,0,10000715
-0,2012-03-28,FP-non GTC,46575,0,16,77.7396,0,10000756
-0,2012-02-19,Auction,43479,0,-99,45.2450,0,10000049
-0,2012-04-02,FP-GTC,41940,0,15,58.8415,0,10000195
-0,2013-08-24,Others,16145,0,11,5.4971,0,10000119
-0,2013-01-10,Auction,106340,15,12,24.1336,0,10000603
-0,2012-09-23,Auction,152801,0,5,70.4024,0,10000384
-0,2013-07-29,FP-GTC,73506,0,16,14.5932,0,10000580
-0,2012-02-16,FP-non GTC,67698,0,5,4.4981,0,10000524
-0,2013-01-06,Others,65,0,13,64.8343,0,10000731
-0,2013-03-02,Auction,43398,0,16,22.6621,0,10000734
-0,2013-10-25,Auction,57990,0,13,64.3384,0,10000961
-0,2012-07-07,FP-GTC,156356,0,16,47.3744,0,10000795
-0,2013-12-19,FP-GTC,67698,0,16,96.9535,0,10000207
-0,2013-11-04,Auction,63861,0,5,34.8173,0,10000964
-0,2013-01-18,FP-non GTC,174053,3,5,94.6036,0,10000393
-0,2013-01-30,Auction,43479,0,16,28.9982,0,10000854
-0,2012-05-16,ABIN,24541,0,15,91.6347,0,10000536
-0,2013-06-03,FP-non GTC,159184,0,15,27.9722,0,10000644
-0,2013-08-24,FP-GTC,2635,0,14,95.7894,0,10000503
-0,2012-07-02,FP-non GTC,16509,0,5,76.9646,0,10000320
-0,2013-09-09,ABIN,80287,0,16,68.9620,0,10000934
-0,2012-04-15,FP-GTC,156356,0,12,95.3640,0,10000269
-0,2013-07-10,FP-GTC,1349,0,13,82.7969,0,10000033
-0,2012-12-02,FP-GTC,963,0,13,31.4196,0,10000717
-0,2013-02-17,Auction,26249,0,14,9.5202,0,10000543
-0,2013-02-17,Others,87118,0,16,77.3899,0,10000647
-0,2012-12-12,Others,6762,0,15,76.3422,0,10000076
-0,2013-07-08,Others,80053,0,5,95.2412,0,10000723
-0,2012-08-19,ABIN,32876,0,12,40.9587,0,10000352
-0,2013-07-24,Auction,314,0,11,78.8147,0,10000996
-0,2013-10-10,Others,57990,0,12,89.8124,0,10000549
-0,2013-11-13,Others,139973,0,16,85.5451,0,10000089
-0,2013-03-29,FP-non GTC,25147,0,16,16.2290,0,10000017
-0,2012-06-04,FP-non GTC,44079,0,-99,0.1588,0,10000758
-0,2012-07-19,FP-non GTC,57784,0,12,84.9441,0,10000553
-0,2012-01-08,FP-GTC,155226,0,15,31.9705,0,10000778
-0,2012-09-13,FP-non GTC,963,0,16,39.6392,0,10000799
-0,2012-12-24,Others,175750,0,15,77.3617,0,10000164
-0,2013-08-31,Auction,57990,0,12,76.7885,0,10000580
-0,2013-08-06,FP-non GTC,156356,0,13,86.0218,0,10000445
-0,2013-05-29,FP-non GTC,11848,0,11,23.0269,0,10000372
-0,2013-05-19,Others,53064,0,13,66.3990,0,10000390
-0,2012-11-01,FP-non GTC,51582,0,14,57.6157,0,10000061
-0,2013-01-01,Auction,1357,0,-99,18.2117,0,10000860
-0,2013-05-17,Others,121153,0,14,79.4534,0,10000155
-0,2013-07-04,FP-GTC,80053,0,5,43.3271,0,10000456
-0,2013-12-17,Others,165888,0,-99,9.8402,0,10000357
-0,2013-11-28,FP-GTC,26262,0,-99,1.4013,0,10000731
-0,2013-02-08,FP-GTC,65,0,11,61.9604,0,10000176
-0,2012-05-17,ABIN,9426,3,12,68.0170,0,10000895
-0,2012-08-29,Auction,159184,0,13,43.7406,0,10000723
-0,2013-10-06,FP-GTC,61323,0,14,85.9325,0,10000692
-0,2013-08-08,FP-GTC,20886,0,14,88.0137,0,10000063
-0,2013-04-06,FP-non GTC,156356,0,16,67.1201,0,10000441
-0,2012-03-09,Auction,1504,0,13,45.8878,0,10000862
-0,2013-12-13,FP-GTC,139973,3,16,58.7879,0,10000947
-0,2013-02-03,ABIN,95173,0,-99,33.2714,0,10000634
-0,2013-02-19,Others,41940,0,5,32.2179,0,10000257
-0,2012-05-09,ABIN,65,0,11,96.0579,0,10000675
-0,2012-10-30,Auction,80135,0,11,55.5335,0,10000288
-0,2013-07-14,ABIN,11554,0,16,9.7012,0,10000210
-0,2012-07-29,Others,11554,0,5,91.5938,0,10000736
-0,2012-06-03,ABIN,80287,0,13,99.7431,0,10000403
-0,2012-10-10,Auction,36250,0,13,14.7056,0,10000561
-0,2013-06-20,ABIN,20485,0,5,23.9120,0,10000798
-0,2013-12-26,Others,11554,0,15,27.2028,0,10000154
-0,2013-05-07,Auction,11848,0,-99,73.7354,0,10000448
-0,2013-12-14,ABIN,61323,0,16,44.6086,0,10000555
-0,2012-08-29,FP-GTC,139973,0,14,34.7404,0,10000686
-0,2013-12-31,FP-GTC,100847,0,15,70.6027,0,10000534
-0,2013-12-13,FP-non GTC,16145,3,12,76.5802,0,10000864
-0,2012-01-24,FP-non GTC,57990,0,-99,54.7904,0,10000325
-0,2013-07-26,Others,20485,0,11,9.3130,0,10000991
-0,2013-09-30,ABIN,31519,0,15,95.7797,0,10000701
-0,2012-03-30,FP-non GTC,88750,0,16,34.6777,0,10000494
-0,2012-05-11,Others,20485,0,16,2.2782,0,10000436
-0,2012-11-21,FP-non GTC,156356,0,13,3.4276,0,10000574
-0,2012-11-19,FP-non GTC,36250,0,16,42.1665,0,10000885
-0,2013-01-07,Others,20485,0,13,29.1424,0,10000931
-0,2013-01-16,ABIN,88750,0,13,87.3858,0,10000276
-0,2012-07-20,ABIN,25147,0,-99,41.9809,0,10000928
-0,2013-06-27,FP-non GTC,75665,0,11,65.3889,0,10000587
-0,2013-12-04,FP-non GTC,45333,0,15,41.7577,0,10000979
-0,2012-05-01,Auction,4943,0,11,76.3978,0,10000385
-0,2013-05-03,Others,66767,15,-99,69.0620,0,10000186
-0,2013-02-14,FP-non GTC,95672,0,14,15.6152,0,10000356
-0,2012-01-01,Others,1357,0,13,16.7271,0,10000263
-0,2012-01-05,FP-non GTC,67703,3,5,32.2005,0,10000141
-0,2013-12-15,Others,2023,0,-99,55.3349,0,10000920
-0,2012-10-02,FP-GTC,314,0,14,95.9217,0,10000647
-0,2012-10-27,Others,64076,0,5,3.4266,0,10000486
-0,2013-10-17,Others,41940,0,15,3.0363,0,10000299
-0,2013-02-09,Others,13836,0,15,37.1746,0,10000799
-0,2013-04-03,FP-GTC,31519,0,15,83.3031,0,10000656
-0,2013-08-30,Others,63861,0,13,37.8620,0,10000958
-0,2012-01-01,Others,1349,0,13,0.7227,0,10000774
-0,2013-03-24,FP-non GTC,87118,0,13,32.5615,0,10000661
-0,2012-12-03,FP-non GTC,216,0,5,40.1447,0,10000923
-0,2012-07-12,FP-GTC,94847,0,5,74.4586,0,10000161
-0,2013-03-18,Others,1161,3,-99,70.1521,0,10000123
-0,2013-11-17,Auction,165888,0,5,16.1344,0,10000029
-0,2013-11-25,FP-non GTC,44079,0,5,18.0996,0,10000461
-0,2012-12-06,FP-GTC,159184,0,15,63.6868,0,10000399
-0,2013-09-27,FP-non GTC,161567,15,15,35.8966,0,10000283
-0,2013-11-02,ABIN,20865,0,5,70.2863,0,10000724
-0,2013-04-29,Auction,314,0,13,53.6862,0,10000676
-0,2013-01-21,Auction,20886,0,-99,32.5548,0,10000052
-0,2012-04-18,FP-GTC,67698,0,14,90.8206,0,10000221
-0,2013-10-14,Others,46575,0,13,92.8392,0,10000187
-0,2013-07-06,FP-GTC,26249,0,-99,54.6729,0,10000651
-0,2012-09-15,Others,16145,0,15,70.0461,0,10000279
-0,2013-11-02,Others,38238,0,11,22.9595,0,10000451
-0,2012-06-03,ABIN,57013,0,11,28.9005,0,10000028
-0,2013-10-08,FP-GTC,20886,0,14,24.6958,0,10000676
-0,2013-04-11,Auction,100847,0,14,42.1878,0,10000645
-0,2013-11-07,Auction,45333,0,13,58.1089,0,10000182
-0,2012-10-18,FP-non GTC,57990,0,15,37.9049,0,10000696
-0,2012-11-28,FP-non GTC,63889,0,5,59.1268,0,10000116
-0,2012-03-02,Auction,45333,0,13,37.4799,0,10000298
-0,2013-07-27,FP-non GTC,75665,0,12,19.4230,0,10000333
-0,2012-12-01,Auction,44079,0,11,95.4596,0,10000163
-0,2012-04-11,FP-non GTC,161567,15,13,47.4582,0,10000082
-0,2013-06-23,ABIN,175750,3,13,23.9511,0,10000560
-0,2012-02-17,Others,46575,0,15,25.7613,0,10000308
-0,2013-05-19,ABIN,223,0,15,64.7127,0,10000850
-0,2012-10-29,FP-non GTC,159184,0,5,15.4188,0,10000995
-0,2012-07-17,Auction,4943,0,11,32.1734,0,10000820
-0,2013-01-03,ABIN,57784,0,5,47.7542,0,10000563
-0,2013-07-17,FP-GTC,150047,3,16,35.0905,0,10000545
-0,2013-07-20,Others,87118,0,14,24.6574,0,10000985
-0,2013-12-14,Others,11848,0,14,80.9504,0,10000318
-0,2013-01-16,Auction,106246,0,12,94.2201,0,10000113
-0,2012-11-04,Others,63861,0,12,71.8354,0,10000985
-0,2012-08-25,FP-non GTC,100847,0,5,68.7418,0,10000581
-0,2013-07-22,Auction,156356,0,15,63.6314,0,10000140
-0,2013-11-04,Auction,67698,0,16,15.6061,0,10000128
-0,2013-03-01,FP-GTC,20485,0,-99,49.2253,0,10000722
-0,2013-12-03,FP-GTC,46575,0,-99,71.5261,0,10000796
-0,2013-03-25,FP-non GTC,60340,0,11,23.0796,0,10000605
-0,2012-11-16,FP-non GTC,99985,0,11,91.6223,0,10000619
-0,2012-05-24,Auction,16509,0,16,97.7044,0,10000002
-0,2012-04-04,ABIN,139973,3,13,16.7393,0,10000923
-0,2012-07-20,FP-GTC,174106,3,13,8.8874,0,10000134
-0,2013-09-12,Auction,33038,15,13,46.7475,0,10000154
-0,2012-06-12,ABIN,57990,0,11,33.1479,0,10000036
-0,2013-08-31,ABIN,46575,0,12,41.0635,0,10000310
-0,2012-12-18,Auction,1349,0,5,14.6328,0,10000296
-0,2012-02-18,ABIN,16509,0,-99,76.3163,0,10000486
-0,2013-07-02,Auction,3838,0,11,95.7402,0,10000963
-0,2012-06-28,Auction,145970,0,16,41.1425,0,10000646
-0,2013-11-03,FP-GTC,12688,0,5,73.9106,0,10000357
-0,2012-03-30,FP-non GTC,16509,0,5,39.5703,0,10000495
-0,2012-08-18,FP-non GTC,75665,0,11,61.4928,0,10000522
-0,2013-07-29,Others,156356,0,16,98.2405,0,10000417
-0,2013-07-19,Others,32876,0,12,68.5148,0,10000306
-0,2012-02-13,ABIN,175750,0,16,64.5444,0,10000274
-0,2013-02-04,FP-non GTC,13836,0,-99,65.7663,0,10000511
-0,2013-07-24,FP-GTC,165888,0,16,91.7808,0,10000330
-0,2012-04-25,Others,41940,0,-99,6.5053,0,10000357
-0,2013-11-28,ABIN,106340,15,12,71.4264,0,10000155
-0,2012-09-16,FP-non GTC,170083,3,-99,80.2442,0,10000690
-0,2012-05-10,FP-GTC,16145,0,14,11.0775,0,10000372
-0,2013-10-28,FP-non GTC,15687,0,16,65.9436,0,10000032
-0,2013-05-17,FP-non GTC,94847,0,13,37.3520,0,10000044
-0,2013-05-31,FP-GTC,223,0,5,93.3181,0,10000365
-0,2012-01-07,FP-non GTC,73506,0,13,13.1225,0,10000218
-0,2013-07-01,FP-GTC,64076,0,15,78.1094,0,10000081
-0,2013-07-26,Others,67698,0,11,14.9245,0,10000775
-0,2013-03-02,Auction,75665,0,5,84.2445,0,10000162
-0,2012-01-13,FP-GTC,1120,3,16,43.8855,0,10000157
-0,2012-06-02,ABIN,53064,0,12,17.3600,0,10000091
-0,2012-02-23,ABIN,31387,3,14,43.1757,0,10000071
-0,2013-10-22,FP-non GTC,158798,0,15,10.9759,0,10000341
-0,2013-05-23,FP-GTC,11848,0,-99,7.5230,0,10000277
-0,2013-07-25,FP-GTC,43479,0,14,13.2620,0,10000500
-0,2012-03-03,FP-non GTC,11848,0,14,95.0683,0,10000164
-0,2012-01-09,ABIN,11554,0,15,6.8777,0,10000785
-0,2013-05-08,ABIN,80287,0,14,95.5320,0,10000595
-0,2013-07-26,FP-GTC,121153,0,12,28.7213,0,10000829
-0,2012-11-27,FP-GTC,31673,0,14,7.8033,0,10000498
-0,2013-06-04,ABIN,43398,0,15,76.0187,0,10000061
-0,2012-10-05,ABIN,15687,0,13,87.1235,0,10000649
-0,2013-06-29,Auction,61323,0,14,78.4000,0,10000765
-0,2012-07-25,ABIN,216,0,12,43.1556,0,10000597
-0,2013-01-03,FP-non GTC,121153,0,12,70.1009,0,10000201
-0,2013-10-22,FP-non GTC,170302,15,12,92.5959,0,10000151
-0,2012-02-03,FP-non GTC,155226,0,12,68.6486,0,10000870
-0,2013-03-30,FP-GTC,139973,0,11,93.3324,0,10000585
-0,2012-11-21,Others,20865,0,11,68.0525,0,10000115
-0,2012-11-09,ABIN,11848,0,13,66.3745,0,10000652
-0,2012-10-22,Auction,2023,0,13,52.4918,0,10000836
-0,2013-01-07,Auction,164261,0,-99,56.4680,0,10000560
-0,2012-01-05,Others,80053,0,-99,34.1305,0,10000905
-0,2012-08-27,FP-GTC,64076,0,14,3.0691,0,10000389
-0,2013-06-23,Others,139973,0,16,89.7673,0,10000636
-0,2013-09-13,Auction,67698,0,15,62.9192,0,10000596
-0,2012-08-18,FP-GTC,43398,0,12,45.3384,0,10000382
-0,2012-10-27,FP-non GTC,1120,3,5,84.6466,0,10000487
-0,2012-05-24,Auction,24541,0,14,56.7041,0,10000366
-0,2013-02-12,Auction,64076,0,13,1.1360,0,10000089
-0,2012-02-19,FP-non GTC,43398,0,14,7.2416,0,10000129
-0,2012-07-30,ABIN,48027,0,11,41.6704,0,10000761
-0,2013-10-04,FP-non GTC,37831,0,16,31.9567,0,10000610
-0,2013-11-18,Auction,223,0,16,21.8155,0,10000079
-0,2012-06-08,Others,80135,0,16,90.0166,0,10000478
-0,2012-10-25,Others,159184,0,16,92.0436,0,10000166
-0,2013-10-22,ABIN,13987,0,15,62.5546,0,10000592
-0,2013-11-01,ABIN,103324,15,16,14.9475,0,10000924
-0,2013-03-21,FP-GTC,1120,3,-99,29.5252,0,10000758
-0,2012-03-13,Others,759,0,16,62.4158,0,10000810
-0,2012-05-14,Auction,95672,0,16,92.0781,0,10000609
-0,2013-09-28,FP-GTC,64076,0,12,37.3599,0,10000487
-0,2013-10-09,ABIN,60606,3,13,48.9467,0,10000007
-0,2013-04-21,ABIN,26249,0,14,67.1520,0,10000257
-0,2013-05-08,FP-GTC,20886,0,13,19.5685,0,10000122
-0,2013-12-29,ABIN,1357,0,12,89.9069,0,10000900
-0,2013-02-14,FP-GTC,63889,0,16,74.5657,0,10000676
-0,2012-02-25,Auction,88750,0,-99,92.3939,0,10000769
-0,2012-06-22,Others,13836,0,14,77.1277,0,10000795
-0,2012-01-30,FP-GTC,43479,0,15,48.2346,0,10000013
-0,2013-08-17,ABIN,57990,3,-99,87.0638,0,10000277
-0,2013-06-24,FP-GTC,1357,0,15,75.0985,0,10000056
-0,2013-04-08,ABIN,57013,0,11,21.4485,0,10000195
-0,2012-04-07,FP-non GTC,24541,0,-99,40.6197,0,10000313
-0,2012-07-26,Others,963,0,16,99.8137,0,10000257
-0,2012-07-18,Others,15115,0,-99,9.9153,0,10000937
-0,2012-01-04,Auction,63889,0,13,84.5004,0,10000467
-0,2013-05-25,Auction,1120,3,14,40.3618,0,10000076
-0,2013-03-30,FP-GTC,38238,0,12,94.9870,0,10000997
-0,2012-05-12,Auction,32876,0,-99,38.5800,0,10000943
-0,2012-11-21,FP-GTC,100847,0,14,7.7495,0,10000588
-0,2013-10-02,Auction,15868,0,5,92.2309,0,10000520
-0,2013-11-05,ABIN,31673,0,-99,75.9838,0,10000344
-0,2013-03-23,FP-non GTC,48904,0,-99,15.4260,0,10000090
-0,2012-12-27,FP-non GTC,43479,0,5,35.7689,0,10000066
-0,2013-01-11,FP-non GTC,223,0,12,5.9423,0,10000174
-0,2012-05-25,Auction,50508,0,16,81.5853,0,10000337
-0,2012-09-08,Others,6762,0,15,11.8473,0,10000718
-0,2013-07-08,Auction,139973,0,5,11.7025,0,10000537
-0,2013-04-23,ABIN,32876,0,12,40.4116,0,10000798
-0,2013-03-22,Auction,2023,0,15,89.8093,0,10000662
-0,2012-11-01,Others,75708,3,11,95.9527,0,10000776
-0,2012-01-29,ABIN,156356,0,16,64.6181,0,10000910
-0,2012-01-20,Others,2635,0,11,81.6196,0,10000893
-0,2012-05-27,FP-non GTC,94847,0,15,87.1713,0,10000121
-0,2013-12-17,Others,156614,0,14,87.7791,0,10000531
-0,2013-03-05,ABIN,80135,0,16,64.8966,0,10000452
-0,2013-01-26,ABIN,156356,0,16,59.0108,0,10000671
-0,2012-01-08,FP-non GTC,57990,0,15,56.3470,0,10000028
-0,2013-08-04,ABIN,13836,0,11,98.5679,0,10000481
-0,2013-04-04,Others,63861,0,13,14.3911,0,10000944
-0,2013-02-15,FP-GTC,2023,0,5,5.5878,0,10000816
-0,2013-05-05,FP-non GTC,175750,0,-99,80.9471,0,10000727
-0,2013-06-24,FP-GTC,38238,0,12,96.7825,0,10000202
-0,2013-11-04,ABIN,45333,0,14,57.9540,0,10000767
-0,2012-02-23,FP-non GTC,57990,0,5,16.4790,0,10000146
-0,2013-01-13,ABIN,158798,0,11,45.5371,0,10000133
-0,2013-11-16,ABIN,73506,0,16,35.7207,0,10000304
-0,2012-11-15,Auction,64076,0,15,66.0059,0,10000375
-0,2012-01-03,ABIN,156614,0,15,84.8759,0,10000095
-0,2012-09-22,Auction,57990,0,14,42.7313,0,10000953
-0,2013-11-03,FP-non GTC,67698,0,11,8.9322,0,10000913
-0,2012-08-31,ABIN,1504,0,5,34.3706,0,10000748
-0,2013-12-13,FP-GTC,15868,0,-99,18.3204,0,10000495
-0,2012-07-20,ABIN,533,0,11,24.9451,0,10000237
-0,2012-07-14,ABIN,41940,0,11,42.9796,0,10000328
-0,2013-11-06,Others,32876,0,15,4.2693,0,10000383
-0,2012-07-02,Auction,41940,0,12,19.2987,0,10000196
-0,2012-06-21,FP-GTC,20865,0,-99,36.5029,0,10000098
-0,2012-09-19,ABIN,159184,0,15,65.9768,0,10000517
-0,2012-02-11,Others,16145,0,14,41.4762,0,10000733
-0,2012-07-19,Auction,63861,0,11,30.7294,0,10000351
-0,2013-05-24,ABIN,1349,0,12,61.0346,0,10000078
-0,2013-06-26,Auction,139973,3,11,49.3849,0,10000537
-0,2013-11-30,Others,1357,0,16,62.0388,0,10000532
-0,2012-11-01,Others,121153,0,5,64.8814,0,10000820
-0,2013-10-20,Auction,20485,0,5,88.9093,0,10000467
-0,2013-04-15,FP-GTC,43479,0,13,72.5969,0,10000352
-0,2012-04-02,FP-GTC,33977,15,16,42.1845,0,10000614
-0,2012-03-20,ABIN,2023,0,16,95.0254,0,10000014
-0,2013-02-21,FP-non GTC,57990,0,5,25.0404,0,10000386
-0,2013-06-02,FP-non GTC,6762,0,12,60.3684,0,10000783
-0,2012-01-07,FP-GTC,759,0,11,80.1720,0,10000952
-0,2012-12-01,FP-non GTC,533,0,12,47.1653,0,10000550
-0,2012-11-03,ABIN,99985,0,16,40.8280,0,10000629
-0,2013-10-15,FP-GTC,64076,0,15,55.9541,0,10000531
-0,2013-12-30,Auction,80135,0,13,13.2503,0,10000425
-0,2012-06-28,FP-GTC,15868,0,12,76.8360,0,10000557
-0,2013-04-07,FP-GTC,10866,0,16,37.7540,0,10000810
-0,2013-08-03,FP-GTC,139255,15,13,70.8704,0,10000514
-0,2012-07-23,ABIN,121153,0,5,65.6362,0,10000166
-0,2013-05-04,FP-GTC,31519,0,5,38.9751,0,10000932
-0,2013-01-27,ABIN,175750,3,-99,81.7736,0,10000529
-0,2013-03-30,ABIN,11554,0,15,37.8703,0,10000509
-0,2012-06-03,FP-GTC,20485,0,-99,62.4729,0,10000404
-0,2012-05-09,FP-non GTC,63861,0,11,35.1334,0,10000175
-0,2013-12-25,FP-non GTC,6762,0,12,95.0363,0,10000101
-0,2012-11-18,FP-GTC,175750,0,5,95.8938,0,10000537
-0,2013-01-07,FP-non GTC,57784,0,14,0.2995,0,10000847
-0,2012-03-11,Auction,63861,0,12,58.1028,0,10000912
-0,2012-03-23,Others,80287,0,14,24.7900,0,10000967
-0,2012-03-19,ABIN,99985,0,16,5.9564,0,10000684
-0,2013-11-24,Auction,20886,0,15,28.1384,0,10000239
-0,2012-04-04,FP-GTC,100847,0,11,91.4446,0,10000743
-0,2013-12-15,ABIN,44079,0,16,39.6860,0,10000476
-0,2013-10-06,FP-GTC,132939,0,12,20.8855,0,10000854
-0,2013-12-07,Others,25147,0,14,16.4357,0,10000198
-0,2012-02-04,Others,158798,0,13,44.5451,0,10000543
-0,2013-11-07,FP-GTC,10866,0,16,8.5249,0,10000105
-0,2012-08-04,FP-GTC,2635,0,12,11.7572,0,10000382
-0,2012-08-14,Auction,152801,0,15,46.6818,0,10000205
-0,2013-02-25,Auction,50508,0,15,37.8443,0,10000644
-0,2013-10-09,ABIN,223,0,13,58.8049,0,10000768
-0,2012-01-10,Auction,20485,0,11,92.2997,0,10000975
-0,2013-05-03,FP-non GTC,37831,0,12,15.9399,0,10000280
-0,2013-11-12,ABIN,73506,0,12,13.9272,0,10000332
-0,2013-08-16,Others,164,0,12,39.2371,0,10000887
-0,2013-10-27,FP-non GTC,15868,0,16,51.4769,0,10000303
-0,2013-04-27,FP-non GTC,80287,0,15,90.8971,0,10000207
-0,2013-11-06,FP-non GTC,11554,0,12,26.0123,0,10000855
-0,2013-11-25,Others,6762,0,14,96.0198,0,10000418
-0,2012-08-09,ABIN,80135,0,15,89.0544,0,10000861
-0,2013-09-28,ABIN,16509,0,12,90.2997,0,10000520
-0,2012-03-19,FP-GTC,43479,0,-99,93.8319,0,10000045
-0,2012-05-25,FP-GTC,139973,0,16,8.9816,0,10000684
-0,2013-04-04,Auction,25147,0,11,43.4557,0,10000801
-0,2013-07-07,Auction,67698,0,14,59.3634,0,10000305
-0,2013-10-27,ABIN,63861,0,14,48.1526,0,10000963
-0,2012-05-31,Auction,130,0,5,1.0560,0,10000357
-0,2012-08-21,Auction,13836,0,16,14.4247,0,10000278
-0,2013-07-24,Auction,15568,15,14,14.5748,0,10000590
-0,2013-04-05,ABIN,12688,0,12,2.0636,0,10000951
-0,2013-02-27,FP-non GTC,73506,0,14,2.7139,0,10000134
-0,2013-06-16,FP-non GTC,164,0,5,59.4198,0,10000745
-0,2012-04-01,Others,20213,0,11,77.8490,0,10000929
-0,2013-12-02,FP-non GTC,155226,0,16,71.1100,0,10000433
-0,2013-06-07,Auction,158798,0,15,98.6517,0,10000320
-0,2012-11-25,ABIN,31519,0,12,68.8974,0,10000726
-0,2013-04-17,FP-GTC,57013,0,14,49.4405,0,10000050
-0,2013-03-18,FP-non GTC,175750,0,13,2.3784,0,10000559
-0,2013-08-13,Others,62179,0,11,36.4940,0,10000652
-0,2012-10-15,ABIN,175750,0,14,42.7269,0,10000560
-0,2012-01-08,Others,6762,0,5,58.5212,0,10000829
-0,2013-05-20,FP-non GTC,216,0,14,16.9644,0,10000104
-0,2012-12-07,Others,51582,0,5,6.0617,0,10000228
-0,2013-12-04,ABIN,57013,0,16,48.2921,0,10000347
-0,2013-12-17,ABIN,759,0,14,12.8117,0,10000161
-0,2012-02-08,FP-non GTC,24760,0,13,11.3190,0,10000938
-0,2013-01-03,FP-non GTC,15568,15,14,0.3508,0,10000964
-0,2013-04-21,Others,87118,0,12,9.5385,0,10000949
-0,2013-11-13,Others,36250,0,12,0.9029,0,10000532
-0,2013-03-18,FP-GTC,164,0,12,3.7727,0,10000237
-0,2013-06-29,Others,15868,0,11,23.3421,0,10000671
-0,2013-12-25,Others,25147,0,5,99.4168,0,10000708
-0,2012-10-06,FP-GTC,533,0,12,40.5069,0,10000295
-0,2013-02-21,ABIN,216,0,14,84.3818,0,10000637
-0,2012-08-27,FP-non GTC,67698,0,16,7.0299,0,10000671
-0,2013-04-13,Others,51582,0,13,67.9730,0,10000778
-0,2013-07-10,FP-GTC,139973,0,13,24.8423,0,10000253
-0,2013-11-02,ABIN,1349,0,14,56.6310,0,10000922
-0,2013-05-11,Others,26249,0,16,49.7279,0,10000610
-0,2013-01-17,FP-non GTC,118687,3,11,34.6913,0,10000002
-0,2012-07-02,Others,148324,15,-99,90.6687,0,10000613
-0,2013-07-14,ABIN,41940,0,-99,60.2733,0,10000145
-0,2013-06-13,Auction,63889,0,11,87.9644,0,10000920
-0,2013-05-05,FP-non GTC,1120,3,16,99.4791,0,10000720
-0,2012-03-30,Others,4943,0,14,56.9643,0,10000105
-0,2012-01-13,FP-non GTC,139973,3,-99,15.8380,0,10000587
-0,2012-09-02,ABIN,12688,0,13,69.4513,0,10000756
-0,2013-08-08,Others,121153,0,11,29.2318,0,10000289
-0,2013-03-09,Others,99985,0,15,14.7661,0,10000453
-0,2012-10-17,FP-non GTC,25147,0,11,96.1909,0,10000054
-0,2012-01-20,Others,73506,0,14,56.0157,0,10000909
-0,2012-04-15,FP-GTC,16145,0,12,94.6354,0,10000551
-0,2013-08-08,Others,3838,0,13,85.7695,0,10000464
-0,2013-01-21,Others,139973,0,13,64.5021,0,10000107
-0,2012-06-17,Others,139973,0,-99,72.8065,0,10000495
-0,2013-12-31,Others,20886,0,15,91.2326,0,10000746
-0,2012-02-11,FP-non GTC,4943,0,14,28.3146,0,10000649
-0,2013-10-01,Auction,155226,0,15,72.2196,0,10000576
-0,2012-08-25,ABIN,223,0,15,17.6408,0,10000661
-0,2012-07-10,Others,67698,0,15,70.9451,0,10000115
-0,2013-12-27,FP-non GTC,26262,0,13,64.5694,0,10000740
-0,2012-02-05,Auction,32996,15,12,9.9521,0,10000833
-0,2012-11-07,FP-non GTC,25147,0,16,83.3886,0,10000494
-0,2014-01-01,Others,45333,0,16,19.2606,0,10000869
-0,2013-10-05,Auction,50508,0,14,39.1465,0,10000904
-0,2012-02-25,FP-GTC,37831,0,11,12.4616,0,10000680
-0,2012-05-16,Others,31519,0,12,66.9268,0,10000548
-0,2013-11-08,FP-GTC,100847,0,11,47.7510,0,10000699
-0,2012-03-15,ABIN,106246,0,16,96.1592,0,10000922
-0,2013-05-15,Auction,64076,0,15,37.7430,0,10000148
-0,2012-11-03,Auction,61323,0,15,14.3851,0,10000353
-0,2012-07-28,ABIN,175750,0,-99,45.5415,0,10000869
-0,2013-06-26,FP-non GTC,95672,0,14,90.7320,0,10000507
-0,2012-09-11,Others,223,0,15,10.5185,0,10000705
-0,2012-09-24,FP-GTC,43479,0,16,3.4611,0,10000991
-0,2013-09-13,FP-GTC,100847,0,-99,51.0213,0,10000871
-0,2013-07-01,Others,132939,0,16,14.6584,0,10000859
-0,2012-06-13,Others,26262,0,-99,46.0446,0,10000041
-0,2013-10-26,Auction,3838,0,12,24.8196,0,10000561
-0,2012-03-05,Others,26262,0,13,66.7748,0,10000859
-0,2012-12-10,FP-GTC,20485,0,12,52.3473,0,10000171
-0,2013-11-15,Others,2023,0,5,84.6955,0,10000451
-0,2013-12-12,ABIN,80135,0,11,95.0112,0,10000718
-0,2013-04-16,ABIN,106340,15,16,14.6367,0,10000822
-0,2012-03-10,Auction,161567,15,16,5.2800,0,10000081
-0,2013-09-19,Auction,11848,0,11,27.6166,0,10000388
-0,2013-12-28,FP-GTC,216,0,15,90.1257,0,10000122
-0,2012-01-26,ABIN,314,0,12,3.5634,0,10000912
-0,2013-12-13,FP-GTC,158798,0,5,41.6652,0,10000485
-0,2013-09-26,FP-non GTC,80135,0,11,20.5746,0,10000138
-0,2012-01-20,Auction,62179,0,14,94.2904,0,10000310
-0,2013-04-20,ABIN,963,0,5,68.3604,0,10000571
-0,2012-02-03,FP-GTC,164,0,11,94.0501,0,10000120
-0,2013-10-14,Others,2023,0,12,84.9792,0,10000371
-0,2013-08-19,FP-GTC,57990,3,14,57.0343,0,10000171
-0,2012-06-18,Others,963,0,12,44.4186,0,10000343
-0,2012-10-18,Others,31519,0,12,50.6753,0,10000875
-0,2013-07-09,Others,36250,0,16,99.5251,0,10000446
-0,2012-01-19,ABIN,38238,0,14,0.1259,0,10000242
-0,2013-05-19,FP-non GTC,13836,0,16,92.7668,0,10000895
-0,2013-01-29,Others,95672,0,5,13.6012,0,10000843
-0,2013-01-30,ABIN,67698,0,5,13.9968,0,10000424
-0,2012-09-02,FP-GTC,223,0,-99,38.0021,0,10000803
-0,2012-05-11,Auction,11848,0,13,7.1910,0,10000964
-0,2013-05-14,FP-non GTC,759,0,16,43.1720,0,10000123
-0,2013-03-06,ABIN,3838,0,16,59.8866,0,10000653
-0,2012-08-23,FP-GTC,20213,0,16,91.3442,0,10000615
-0,2013-03-24,FP-GTC,4943,0,-99,18.4365,0,10000718
-0,2012-01-26,FP-non GTC,11554,0,14,41.6729,0,10000174
-0,2013-10-07,Others,108782,15,14,14.9130,0,10000643
-0,2013-07-25,Others,13987,0,-99,28.7021,0,10000154
-0,2013-11-08,FP-non GTC,31673,0,13,83.8884,0,10000059
-0,2013-01-06,ABIN,82494,15,12,74.4582,0,10000023
-0,2013-01-21,FP-non GTC,13987,0,11,72.4276,0,10000541
-0,2013-11-22,Auction,67698,0,11,53.1908,0,10000553
-0,2012-02-02,FP-non GTC,175750,0,12,79.1160,0,10000121
-0,2012-10-14,FP-GTC,24760,0,11,70.5894,0,10000677
-0,2012-10-20,ABIN,73506,0,11,3.7214,0,10000076
-0,2013-12-01,Others,16145,0,15,54.8674,0,10000092
-0,2012-11-07,FP-non GTC,53064,0,14,53.4385,0,10000267
-0,2012-04-20,FP-non GTC,53064,0,15,76.5825,0,10000551
-0,2012-11-18,Auction,50677,0,11,44.5149,0,10000924
-0,2013-07-31,FP-GTC,51582,0,14,86.5325,0,10000652
-0,2012-12-25,Others,31387,3,13,79.7624,0,10000495
-0,2012-08-10,ABIN,2635,0,12,54.5975,0,10000453
-0,2012-06-09,Others,31519,0,15,20.1828,0,10000226
-0,2012-09-16,Others,44079,0,16,46.0626,0,10000193
-0,2012-04-23,Others,16145,0,11,60.2064,0,10000058
-0,2012-01-03,Others,164,0,15,94.1749,0,10000509
-0,2012-05-16,FP-GTC,13987,0,15,5.6834,0,10000052
-0,2012-07-24,FP-non GTC,16145,0,15,8.1625,0,10000036
-0,2013-01-01,FP-non GTC,32996,15,15,75.5057,0,10000393
-0,2013-01-28,FP-non GTC,155226,0,16,49.2395,0,10000966
-0,2013-03-19,FP-GTC,20485,0,5,57.4433,0,10000138
-0,2013-02-03,FP-GTC,57784,0,5,11.8152,0,10000731
-0,2012-01-02,Others,53064,0,-99,45.8162,0,10000785
-0,2013-02-23,ABIN,16145,0,14,25.1092,0,10000582
-0,2013-04-19,Auction,159184,0,12,85.3510,0,10000422
-0,2012-10-12,FP-non GTC,139973,0,11,95.7331,0,10000791
-0,2012-04-20,FP-non GTC,87118,0,16,15.2148,0,10000621
-0,2013-04-30,Auction,53064,0,5,89.5035,0,10000377
-0,2013-10-24,FP-non GTC,73506,0,12,84.0199,0,10000503
-0,2012-01-08,Auction,63861,0,13,74.8860,0,10000589
-0,2012-05-21,Others,63861,3,16,63.7247,0,10000730
-0,2013-01-29,Auction,24541,0,15,22.2189,0,10000167
-0,2012-02-08,Auction,533,0,5,53.5867,0,10000564
-0,2012-08-11,FP-non GTC,174053,3,13,0.5792,0,10000568
-0,2012-07-05,FP-GTC,20886,0,12,31.1464,0,10000806
-0,2012-01-08,Auction,156356,0,5,7.4293,0,10000638
-0,2012-02-14,Others,65,0,14,62.6745,0,10000253
-0,2013-03-25,FP-non GTC,48027,0,-99,53.4890,0,10000899
-0,2012-09-30,Others,57013,0,12,75.9109,0,10000274
-0,2013-09-01,ABIN,32876,0,12,75.0498,0,10000086
-0,2013-11-11,FP-GTC,164,0,12,36.6051,0,10000521
-0,2012-02-18,FP-GTC,57013,0,12,14.3117,0,10000763
-0,2013-11-23,ABIN,139973,0,15,58.3344,0,10000787
-0,2012-03-21,FP-non GTC,41940,0,15,38.2502,0,10000342
-0,2012-02-24,Auction,57990,0,5,73.5795,0,10000175
-0,2013-07-20,Auction,88750,0,-99,58.2532,0,10000122
-0,2012-10-17,Others,43479,0,13,67.9585,0,10000516
-0,2012-11-03,FP-non GTC,118687,3,13,90.9173,0,10000957
-0,2012-10-22,FP-GTC,60340,0,14,68.5877,0,10000502
-0,2013-04-25,FP-GTC,11554,0,5,60.8280,0,10000074
-0,2012-10-23,FP-GTC,24760,0,13,6.0732,0,10000138
-0,2013-04-29,ABIN,175750,0,11,8.0341,0,10000077
-0,2012-05-06,Auction,106340,15,14,8.0384,0,10000234
-0,2012-07-15,ABIN,57784,0,14,27.1882,0,10000729
-0,2012-10-15,Auction,67698,0,-99,63.1790,0,10000226
-0,2013-03-22,FP-non GTC,57013,0,14,27.7251,0,10000658
-0,2013-08-12,FP-non GTC,11848,0,13,38.5597,0,10000020
-0,2012-07-14,FP-GTC,80053,0,-99,81.0002,0,10000850
-0,2012-08-12,FP-non GTC,38238,0,11,58.6987,0,10000377
-0,2013-09-09,FP-GTC,106246,0,11,5.0352,0,10000594
-0,2013-02-11,FP-non GTC,57784,0,12,80.8420,0,10000657
-0,2013-03-09,Others,175750,0,14,87.9884,0,10000152
-0,2013-12-18,FP-non GTC,20485,0,14,74.4627,0,10000272
-0,2013-03-16,FP-non GTC,106246,0,5,64.1494,0,10000459
-0,2012-09-06,FP-non GTC,80135,0,13,30.1506,0,10000269
-0,2013-06-18,FP-non GTC,80135,0,-99,75.5826,0,10000706
-0,2013-03-16,FP-GTC,156614,0,15,36.7157,0,10000761
-0,2012-11-07,Others,1349,0,15,96.7575,0,10000612
-0,2012-11-25,ABIN,95173,0,5,61.1186,0,10000364
-0,2013-08-10,ABIN,31673,0,11,63.4095,0,10000421
-0,2012-02-17,ABIN,88750,0,14,87.8624,0,10000306
-0,2013-03-29,ABIN,139973,0,16,9.1685,0,10000089
-0,2012-12-27,ABIN,121153,0,12,88.7482,0,10000726
-0,2013-10-13,FP-GTC,67698,0,16,20.8225,0,10000504
-0,2012-09-26,ABIN,2023,0,11,83.6499,0,10000641
-0,2012-08-01,Others,139973,0,-99,1.0542,0,10000202
-0,2012-03-02,ABIN,113802,15,5,23.4439,0,10000068
-0,2012-10-25,Others,10866,0,11,60.7453,0,10000770
-0,2013-01-07,FP-non GTC,20485,0,16,16.4177,0,10000698
-0,2012-03-18,FP-non GTC,46575,0,15,55.2768,0,10000962
-0,2012-07-09,FP-GTC,106246,0,14,99.6346,0,10000563
-0,2012-09-26,FP-GTC,314,0,13,45.8302,0,10000769
-0,2012-01-14,FP-non GTC,15868,0,16,47.5496,0,10000869
-0,2012-01-01,ABIN,63861,0,13,76.1393,0,10000028
-0,2012-08-15,FP-GTC,26249,0,5,40.7448,0,10000419
-0,2012-09-24,FP-non GTC,43398,0,12,54.0864,0,10000688
-0,2013-10-13,Others,533,0,-99,58.0312,0,10000658
-0,2012-07-13,FP-GTC,148324,15,11,71.6067,0,10000006
-0,2012-01-21,FP-GTC,32996,15,11,86.8077,0,10000928
-0,2012-04-04,Others,63861,3,16,85.2671,0,10000648
-0,2012-10-20,FP-non GTC,31519,0,13,99.1930,0,10000469
-0,2013-12-28,FP-GTC,1504,0,5,14.9821,0,10000342
-0,2013-06-19,Others,25147,0,5,77.4671,0,10000212
-0,2013-05-04,Others,13836,0,11,13.4655,0,10000331
-0,2012-06-28,ABIN,145970,0,15,74.3304,0,10000900
-0,2012-12-24,FP-non GTC,156614,0,12,79.4508,0,10000743
-0,2013-07-17,FP-GTC,139973,0,13,30.0555,0,10000949
-0,2013-11-23,Others,31519,3,11,9.0845,0,10000883
-0,2013-01-15,Others,164261,0,12,31.5563,0,10000029
-0,2012-04-15,FP-non GTC,20213,0,-99,55.6917,0,10000513
-0,2013-01-20,FP-non GTC,24760,0,15,71.7279,0,10000274
-0,2012-11-13,FP-GTC,95173,0,5,95.7043,0,10000418
-0,2012-09-03,ABIN,57990,0,12,1.5845,0,10000703
-0,2012-01-02,FP-non GTC,57990,3,16,12.3706,0,10000090
-0,2013-12-06,Auction,67703,3,5,70.4134,0,10000753
-0,2012-06-09,Others,50677,0,12,88.7297,0,10000990
-0,2012-05-17,ABIN,31519,0,14,37.7070,0,10000326
-0,2012-03-03,Others,156356,0,16,50.7844,0,10000693
-0,2012-05-21,Auction,44079,0,5,86.7590,0,10000795
-0,2013-10-15,Auction,26262,0,15,56.3327,0,10000077
-0,2013-12-27,ABIN,20213,0,13,96.1815,0,10000195
-0,2012-02-15,ABIN,314,0,5,6.5869,0,10000753
-0,2012-08-17,FP-non GTC,6762,0,15,41.0782,0,10000797
-0,2013-04-29,Auction,145970,0,15,11.0876,0,10000208
-0,2013-10-27,Auction,80053,0,14,81.7435,0,10000310
-0,2012-12-21,FP-GTC,63861,0,13,57.9491,0,10000971
-0,2013-07-29,ABIN,45333,0,12,33.5405,0,10000475
-0,2012-04-18,Auction,64076,0,13,83.3621,0,10000697
-0,2012-12-14,FP-non GTC,63889,0,16,0.9640,0,10000607
-0,2013-06-07,Auction,145970,0,13,6.1316,0,10000450
-0,2012-11-27,Others,164,0,15,53.7718,0,10000305
-0,2012-08-03,ABIN,61323,0,15,62.2945,0,10000214
-0,2012-07-07,ABIN,75665,0,15,77.0551,0,10000094
-0,2012-08-06,FP-non GTC,99985,0,13,39.1253,0,10000609
-0,2012-10-01,Auction,759,0,15,69.1790,0,10000199
-0,2012-03-22,Auction,20865,0,5,40.7800,0,10000392
-0,2012-02-06,FP-non GTC,132939,0,11,66.8705,0,10000207
-0,2012-10-17,FP-non GTC,63861,3,14,35.8018,0,10000096
-0,2013-02-21,Others,41940,0,5,87.4721,0,10000858
-0,2012-11-16,FP-GTC,57784,0,15,96.2799,0,10000537
-0,2012-01-10,Others,158666,15,11,7.4077,0,10000190
-0,2013-10-29,FP-non GTC,65,0,-99,21.3439,0,10000182
-0,2013-07-30,Auction,103178,15,-99,94.8679,0,10000943
-0,2012-08-05,FP-non GTC,175750,0,14,10.0980,0,10000212
-0,2012-07-29,ABIN,24760,0,13,44.5270,0,10000168
-0,2013-04-27,FP-GTC,57990,3,-99,75.6664,0,10000878
-0,2012-12-06,FP-GTC,37831,0,11,57.1191,0,10000090
-0,2013-08-29,FP-GTC,164,0,16,75.2345,0,10000689
-0,2012-08-30,FP-GTC,63861,0,12,21.2099,0,10000643
-0,2013-05-30,ABIN,57013,0,15,28.8410,0,10000290
-0,2013-10-17,ABIN,50677,0,-99,16.3836,0,10000705
-0,2012-04-25,FP-GTC,67698,0,11,52.3502,0,10000257
-0,2013-08-18,ABIN,759,0,16,23.8258,0,10000911
-0,2013-02-24,Others,13836,0,-99,40.6826,0,10000399
-0,2012-07-22,FP-non GTC,158798,0,15,94.1886,0,10000705
-0,2012-08-05,FP-GTC,100847,0,-99,74.1500,0,10000797
-0,2012-12-30,Auction,31519,0,11,69.1499,0,10000576
-0,2013-05-18,Auction,63864,3,14,44.5408,0,10000232
-0,2012-11-01,ABIN,53064,0,15,19.9817,0,10000065
-0,2013-06-25,FP-GTC,156614,0,16,91.4652,0,10000326
-0,2013-07-11,FP-non GTC,10058,3,5,92.6671,0,10000787
-0,2013-01-19,FP-non GTC,13987,0,12,49.0313,0,10000556
-0,2013-06-17,Auction,94847,0,13,95.5377,0,10000047
-0,2012-12-23,Others,64076,0,-99,50.1173,0,10000063
-0,2013-01-07,FP-non GTC,67698,0,5,6.5979,0,10000412
-0,2012-01-20,ABIN,175750,3,16,79.6931,0,10000270
-0,2013-01-14,Others,95672,3,12,31.7315,0,10000044
-0,2013-02-22,Others,26249,0,16,43.7969,0,10000634
-0,2013-04-13,FP-non GTC,156614,0,16,79.3213,0,10000321
-0,2013-09-18,FP-non GTC,43398,0,5,17.3702,0,10000621
-0,2013-09-05,ABIN,67703,3,11,55.6439,0,10000784
-0,2013-05-25,FP-GTC,43398,0,15,73.0548,0,10000921
-0,2012-10-06,Others,100847,0,16,77.0258,0,10000606
-0,2013-11-08,Others,164261,0,11,16.6959,0,10000136
-0,2012-04-13,FP-non GTC,3838,0,5,68.4347,0,10000451
-0,2012-07-03,FP-GTC,63861,0,-99,89.5962,0,10000893
-0,2012-02-21,Auction,40059,3,-99,84.7455,0,10000613
-0,2012-02-25,FP-non GTC,759,0,13,23.9258,0,10000826
-0,2013-10-30,FP-non GTC,57990,0,16,88.4859,0,10000782
-0,2013-10-12,FP-non GTC,57784,0,14,85.8986,0,10000220
-0,2012-08-15,FP-GTC,223,0,12,32.1974,0,10000316
-0,2012-10-18,FP-GTC,15115,0,16,52.5243,0,10000353
-0,2013-11-09,FP-GTC,20886,0,13,94.0128,0,10000690
-0,2012-09-27,ABIN,31673,0,12,24.1332,0,10000404
-0,2012-05-08,FP-GTC,95672,0,16,88.8853,0,10000984
-0,2013-01-14,Others,95173,0,12,86.3987,0,10000501
-0,2012-12-02,Auction,152801,0,-99,71.8799,0,10000021
-0,2012-03-13,ABIN,31519,3,15,96.8998,0,10000188
-0,2013-06-15,Auction,121153,0,14,66.2729,0,10000101
-0,2012-04-11,Others,63861,0,15,88.0097,0,10000536
-0,2013-01-25,FP-non GTC,1357,0,16,49.8991,0,10000637
-0,2013-10-17,FP-GTC,106246,0,15,92.2507,0,10000046
-0,2012-10-04,Others,4943,0,13,88.8498,0,10000215
-0,2013-05-14,Auction,314,0,13,77.3441,0,10000900
-0,2013-03-17,FP-non GTC,759,0,16,17.9719,0,10000759
-0,2013-01-20,FP-non GTC,51582,0,16,3.6644,0,10000961
-0,2013-03-20,FP-non GTC,57990,0,12,9.3680,0,10000846
-0,2012-06-06,FP-non GTC,165888,0,16,9.4584,0,10000312
-0,2012-12-13,Auction,43479,0,16,84.0632,0,10000349
-0,2012-05-13,FP-non GTC,139973,3,5,37.3510,0,10000790
-0,2013-03-27,Auction,48027,0,5,75.7167,0,10000424
-0,2013-08-08,FP-GTC,37831,0,-99,27.7800,0,10000637
-0,2012-04-06,ABIN,314,0,12,18.3274,0,10000749
-0,2013-10-11,FP-non GTC,57013,0,14,39.1868,0,10000599
-0,2012-01-18,FP-GTC,41940,0,12,94.5165,0,10000102
-0,2013-05-20,ABIN,50677,0,16,25.6136,0,10000423
-0,2012-06-24,FP-GTC,963,0,13,20.9442,0,10000069
-0,2013-04-01,ABIN,75665,0,12,65.3307,0,10000543
-0,2012-02-05,ABIN,50508,0,-99,69.4516,0,10000623
-0,2012-06-24,ABIN,314,0,5,13.8612,0,10000347
-0,2013-12-07,Others,1349,0,16,10.8619,0,10000784
-0,2013-03-01,ABIN,73506,0,11,7.7818,0,10000882
-0,2012-08-24,Auction,45333,0,16,94.9583,0,10000495
-0,2012-08-04,FP-non GTC,20886,0,15,69.7300,0,10000428
-0,2012-11-08,Auction,31519,0,15,41.4423,0,10000431
-0,2013-08-07,FP-GTC,156356,0,13,44.6853,0,10000305
-0,2013-09-25,ABIN,43398,0,16,46.5536,0,10000387
-0,2012-04-26,FP-non GTC,95672,0,13,30.9202,0,10000591
-0,2012-03-30,Others,159184,0,12,58.3993,0,10000381
-0,2012-04-25,Auction,73506,0,14,46.1501,0,10000010
-0,2012-09-13,FP-GTC,20485,0,5,2.4112,0,10000959
-0,2013-09-27,ABIN,80287,0,-99,64.4361,0,10000770
-0,2013-01-17,Others,36250,0,11,73.5773,0,10000948
-0,2012-07-07,Auction,103324,15,16,4.8559,0,10000123
-0,2013-08-15,ABIN,13836,0,16,25.8438,0,10000851
-0,2012-07-07,Others,12688,0,5,3.4934,0,10000096
-0,2012-06-03,Auction,80135,0,12,17.8258,0,10000144
-0,2013-12-24,FP-GTC,60340,0,11,22.6409,0,10000341
-0,2012-10-06,ABIN,36250,0,5,7.4322,0,10000384
-0,2013-01-03,Auction,175750,0,15,14.1976,0,10000366
-0,2013-04-15,Auction,88750,0,11,98.6057,0,10000584
-0,2013-06-03,FP-non GTC,1349,0,15,11.0419,0,10000866
-0,2013-01-30,FP-GTC,2635,0,14,54.5175,0,10000959
-0,2013-08-27,FP-non GTC,26249,0,13,50.0365,0,10000289
-0,2012-05-14,FP-non GTC,9426,3,5,60.0224,0,10000730
-0,2012-10-16,FP-non GTC,16145,0,-99,53.0896,0,10000195
-0,2012-09-24,FP-GTC,57013,0,14,81.7929,0,10000498
-0,2012-08-20,FP-non GTC,25147,0,-99,30.7605,0,10000242
-0,2012-02-19,Others,94847,0,13,20.6395,0,10000707
-0,2013-11-14,ABIN,50508,0,15,91.9217,0,10000208
-0,2012-07-10,Others,95672,0,13,21.1886,0,10000762
-0,2012-03-14,ABIN,44079,0,13,89.6838,0,10000227
-0,2012-01-17,FP-non GTC,314,0,14,44.3066,0,10000251
-0,2012-02-27,FP-non GTC,15868,0,14,80.8427,0,10000741
-0,2012-10-23,Others,1161,3,5,54.0104,0,10000289
-0,2013-06-12,Others,53064,0,12,87.1336,0,10000871
-0,2013-03-06,Auction,34273,100,15,6.7044,0,10000269
-0,2012-06-16,Others,533,0,14,43.7104,0,10000172
-0,2012-12-19,FP-GTC,15868,0,15,94.1727,0,10000366
-0,2012-07-21,ABIN,121153,0,12,42.3437,0,10000688
-0,2013-08-19,FP-non GTC,65,0,11,3.0275,0,10000055
-0,2012-08-08,Auction,15687,0,14,72.9505,0,10000725
-0,2013-12-13,FP-GTC,4943,0,-99,37.7781,0,10000932
-0,2013-05-18,FP-GTC,3838,0,14,45.1642,0,10000370
-0,2012-01-31,Auction,25147,0,14,78.6580,0,10000721
-0,2013-12-05,FP-GTC,62179,0,12,76.4692,0,10000423
-0,2012-03-07,Auction,175750,0,15,19.5357,0,10000053
-0,2013-09-30,Others,100847,0,12,38.2299,0,10000772
-0,2013-01-07,Others,15115,0,12,82.5099,0,10000864
-0,2013-11-04,FP-non GTC,57013,0,-99,61.3731,0,10000335
-0,2012-08-11,FP-non GTC,31673,0,12,12.7001,0,10000303
-0,2012-12-30,ABIN,156356,0,14,99.3562,0,10000427
-0,2013-05-19,Others,113593,15,11,31.1634,0,10000361
-0,2013-06-03,ABIN,314,0,13,32.6968,0,10000373
-0,2013-06-29,Auction,15808,15,12,18.3967,0,10000484
-0,2012-06-10,FP-GTC,61323,0,13,10.8143,0,10000403
-0,2012-01-29,FP-GTC,1349,0,15,0.4358,0,10000639
-0,2013-05-30,FP-non GTC,759,0,14,19.5920,0,10000116
-0,2013-07-28,ABIN,15868,0,14,59.7123,0,10000668
-0,2013-03-23,Auction,165888,0,12,67.1201,0,10000063
-0,2012-05-29,FP-GTC,118687,3,-99,98.2030,0,10000886
-0,2013-09-15,FP-non GTC,156614,0,5,43.1795,0,10000302
-0,2012-05-21,FP-GTC,223,0,-99,66.2850,0,10000118
-0,2012-04-03,FP-non GTC,26262,0,12,17.5241,0,10000440
-0,2012-05-22,ABIN,44079,0,16,25.2405,0,10000183
-0,2014-01-01,Others,50508,0,16,92.5843,0,10000911
-0,2012-04-08,Others,16145,3,12,90.2242,0,10000825
-0,2013-11-27,ABIN,95672,0,16,90.9380,0,10000829
-0,2013-03-22,Auction,41940,0,15,72.0648,0,10000795
-0,2012-04-06,Others,64076,0,14,86.9185,0,10000356
-0,2013-05-02,Others,156356,0,11,41.6871,0,10000122
-0,2012-04-02,Others,132939,0,12,75.8372,0,10000395
-0,2012-06-29,ABIN,26262,0,13,83.6043,0,10000120
-0,2012-07-06,FP-GTC,11554,0,16,38.4329,0,10000352
-0,2012-06-10,ABIN,156614,0,-99,94.1812,0,10000280
-0,2012-04-05,Auction,164261,0,12,0.9691,0,10000480
-0,2012-11-02,Auction,31519,0,11,98.0505,0,10000893
-0,2012-05-20,Auction,50508,0,15,61.9439,0,10000363
-0,2012-03-31,Auction,80135,0,16,11.1543,0,10000692
-0,2012-07-01,FP-non GTC,63889,0,-99,86.6557,0,10000402
-0,2013-01-04,ABIN,41940,0,-99,48.4505,0,10000296
-0,2013-02-10,FP-non GTC,63861,0,13,60.3535,0,10000214
-0,2012-03-25,Others,73506,0,11,74.2113,0,10000801
-0,2013-07-23,FP-non GTC,53064,0,16,69.2368,0,10000542
-0,2012-04-09,Auction,11848,0,15,76.1720,0,10000555
-0,2012-08-01,Others,12688,0,5,85.4589,0,10000130
-0,2012-07-15,FP-non GTC,31519,3,-99,17.8485,0,10000045
-0,2013-10-06,ABIN,1349,0,11,52.2780,0,10000622
-0,2012-10-26,Auction,25147,0,13,60.9912,0,10000379
-0,2013-01-04,FP-non GTC,37831,0,15,61.0134,0,10000785
-0,2012-03-24,Auction,139973,0,-99,8.1533,0,10000487
-0,2012-12-30,Auction,6762,0,-99,77.8179,0,10000645
-0,2012-10-03,ABIN,103178,15,12,15.2201,0,10000824
-0,2012-04-18,FP-non GTC,50508,0,14,11.9715,0,10000012
-0,2012-09-30,FP-non GTC,31673,0,12,90.0864,0,10000281
-0,2013-03-24,ABIN,4943,0,12,11.0294,0,10000785
-0,2012-05-02,Others,87118,0,14,52.6549,0,10000924
-0,2013-05-03,FP-GTC,24760,0,5,8.9077,0,10000758
-0,2012-08-05,FP-non GTC,38238,0,11,0.8463,0,10000066
-0,2012-08-15,Auction,67698,0,11,23.6278,0,10000085
-0,2013-04-04,Auction,164261,0,13,52.3718,0,10000965
-0,2013-04-18,ABIN,25147,0,14,71.7687,0,10000157
-0,2013-11-09,ABIN,43479,0,12,39.9269,0,10000024
-0,2013-05-02,FP-non GTC,175750,0,12,17.6149,0,10000862
-0,2012-09-01,ABIN,50508,0,14,99.1518,0,10000746
-0,2013-02-23,FP-non GTC,139255,15,-99,74.6182,0,10000447
-0,2012-01-19,FP-non GTC,6762,0,13,0.1297,0,10000189
-0,2012-11-10,Auction,113802,15,-99,11.9583,0,10000596
-0,2012-12-11,FP-non GTC,95173,0,5,67.9416,0,10000582
-0,2012-01-05,ABIN,2635,0,11,72.3723,0,10000375
-0,2012-05-04,FP-non GTC,43479,0,13,79.1307,0,10000944
-0,2013-07-20,ABIN,63861,0,13,86.7155,0,10000274
-0,2013-08-25,FP-GTC,80053,0,5,11.7108,0,10000059
-0,2012-10-24,ABIN,38238,0,5,79.2964,0,10000388
-0,2012-01-10,ABIN,73506,0,5,3.8878,0,10000895
-0,2013-07-11,Auction,174106,3,12,46.7087,0,10000683
-0,2013-06-16,FP-GTC,61323,0,15,64.0224,0,10000121
-0,2013-06-23,ABIN,63889,0,5,59.2766,0,10000466
-0,2013-01-25,Others,156356,0,14,79.9992,0,10000611
-0,2013-10-09,Auction,50508,0,16,70.9682,0,10000009
-0,2013-05-19,Auction,25147,0,16,8.0973,0,10000731
-0,2012-10-14,Auction,67698,0,5,49.0497,0,10000605
-0,2012-06-08,Auction,65,0,14,76.5149,0,10000165
-0,2012-12-21,FP-GTC,15868,0,5,45.9224,0,10000215
-0,2012-12-02,Auction,1357,0,15,8.9337,0,10000730
-0,2013-11-27,ABIN,24760,0,11,62.4675,0,10000932
-0,2013-11-24,Auction,95173,0,16,16.7927,0,10000795
-0,2013-11-30,ABIN,164,0,11,10.8897,0,10000914
-0,2012-12-22,Auction,15868,0,15,7.4153,0,10000540
-0,2012-05-13,FP-non GTC,26249,0,5,58.8647,0,10000898
-0,2013-06-22,ABIN,95672,3,12,71.8628,0,10000026
-0,2013-05-22,ABIN,63861,0,11,71.8836,0,10000650
-0,2012-11-17,FP-GTC,32876,0,12,35.6019,0,10000959
-0,2013-04-13,Auction,38238,0,5,21.7443,0,10000036
-0,2012-09-09,FP-non GTC,87118,0,14,46.2858,0,10000512
-0,2013-08-23,FP-non GTC,41940,0,11,98.4779,0,10000844
-0,2012-09-02,FP-non GTC,43398,0,12,91.8543,0,10000917
-0,2012-11-23,Others,145970,0,11,43.8631,0,10000997
-0,2012-01-22,FP-non GTC,164261,0,16,74.9772,0,10000091
-0,2012-05-02,ABIN,152801,0,13,15.2794,0,10000649
-0,2013-08-14,Others,50677,0,13,26.5329,0,10000275
-0,2012-06-04,Others,156614,0,14,42.7810,0,10000347
-0,2012-08-04,FP-non GTC,145970,0,15,92.6953,0,10000622
-0,2012-08-15,Auction,32876,0,16,35.2015,0,10000533
-0,2012-11-05,Auction,95672,0,11,0.5233,0,10000965
-0,2013-10-12,ABIN,165888,0,11,27.3564,0,10000036
-0,2012-10-22,ABIN,20485,0,14,28.6742,0,10000448
-0,2012-07-02,Auction,20213,0,15,81.3332,0,10000484
-0,2013-01-13,Others,152801,0,11,3.3406,0,10000124
-0,2012-05-17,Others,132939,0,15,55.3503,0,10000086
-0,2013-02-05,Others,48904,0,15,25.2340,0,10000246
-0,2013-09-13,FP-GTC,32876,0,-99,10.7672,0,10000729
-0,2012-06-14,ABIN,20865,0,14,47.9841,0,10000406
-0,2013-02-11,Auction,152801,0,5,86.4355,0,10000050
-0,2012-12-13,Auction,50508,0,5,72.5114,0,10000715
-0,2013-11-30,Auction,3838,0,-99,6.8517,0,10000706
-0,2013-03-15,FP-non GTC,20865,0,14,93.3040,0,10000926
-0,2012-08-12,FP-GTC,57784,0,-99,18.4045,0,10000231
-0,2013-10-19,Others,1161,3,14,79.1353,0,10000561
-0,2013-02-17,Others,31519,0,-99,49.0409,0,10000904
-0,2012-03-30,FP-non GTC,314,0,11,93.6413,0,10000041
-0,2012-08-26,Others,175750,0,-99,46.6557,0,10000719
-0,2012-12-26,Auction,139973,0,15,6.9809,0,10000044
-0,2013-01-30,Others,159184,0,13,45.4599,0,10000131
-0,2012-04-27,Auction,48904,0,5,16.3472,0,10000157
-0,2012-11-10,ABIN,16145,0,13,0.4693,0,10000668
-0,2012-05-13,Auction,121153,0,5,24.6420,0,10000314
-0,2012-02-01,FP-GTC,6762,0,13,5.9417,0,10000851
-0,2012-07-16,Auction,95672,0,-99,11.9934,0,10000828
-0,2012-09-21,Auction,106246,0,5,39.0057,0,10000826
-0,2012-10-20,FP-GTC,314,0,16,5.5339,0,10000011
-0,2013-04-29,FP-non GTC,100847,0,13,3.3872,0,10000840
-0,2013-02-16,FP-non GTC,43398,0,13,3.4171,0,10000106
-0,2013-09-11,FP-GTC,63889,0,12,15.4662,0,10000004
-0,2012-12-18,Others,80287,0,15,71.3090,0,10000722
-0,2013-09-14,FP-non GTC,20865,0,-99,98.0987,0,10000591
-0,2013-01-27,FP-non GTC,57990,3,14,84.9483,0,10000304
-0,2013-02-07,Others,95672,0,14,9.7034,0,10000894
-0,2013-03-24,FP-non GTC,38238,0,-99,98.5328,0,10000286
-0,2012-01-05,Others,164,0,15,21.3336,0,10000151
-0,2012-11-14,Auction,57990,0,12,91.1773,0,10000087
-0,2013-03-24,Auction,13987,0,13,5.9953,0,10000551
-0,2013-12-26,FP-GTC,32876,0,11,86.1904,0,10000012
-0,2013-06-23,FP-non GTC,1504,0,12,9.4689,0,10000720
-0,2012-01-11,FP-GTC,57013,0,13,96.6705,0,10000790
-0,2012-08-30,FP-non GTC,62179,0,5,22.4244,0,10000403
-0,2012-09-01,ABIN,61323,0,12,30.1513,0,10000283
-0,2013-04-20,Auction,16145,3,12,77.9327,0,10000043
-0,2013-10-03,Others,963,0,14,82.3371,0,10000002
-0,2012-10-23,ABIN,45333,0,16,86.7017,0,10000180
-0,2013-10-09,ABIN,63889,0,-99,33.9942,0,10000485
-0,2013-04-28,ABIN,73506,0,5,22.6563,0,10000070
-0,2013-07-07,ABIN,57990,0,11,11.7659,0,10000834
-0,2012-01-04,FP-non GTC,165888,0,16,78.7873,0,10000743
-0,2012-05-05,ABIN,158666,15,16,55.1036,0,10000372
-0,2012-08-04,ABIN,43479,0,13,61.3469,0,10000425
-0,2013-02-18,Others,161567,15,5,43.5109,0,10000315
-0,2013-08-09,FP-GTC,161567,15,14,0.9756,0,10000420
-0,2012-04-30,Others,65,0,12,31.5741,0,10000858
-0,2013-10-16,FP-non GTC,16509,0,15,8.4323,0,10000376
-0,2013-11-06,FP-GTC,15115,0,12,91.7204,0,10000495
-0,2012-04-13,ABIN,37831,0,16,86.9150,0,10000947
-0,2013-04-21,FP-GTC,95672,0,-99,60.6048,0,10000052
-0,2013-12-04,FP-GTC,121153,0,13,9.3639,0,10000384
-0,2012-08-03,Others,50508,0,13,74.2474,0,10000103
-0,2013-12-29,FP-GTC,1357,0,5,46.9382,0,10000987
-0,2012-11-01,Others,46575,0,13,54.4520,0,10000247
-0,2012-06-30,FP-non GTC,26249,0,5,74.2755,0,10000641
-0,2012-06-20,Auction,26262,0,11,11.9173,0,10000418
-0,2012-07-24,Others,80287,0,14,49.7794,0,10000590
-0,2012-08-11,FP-GTC,26262,0,11,16.2910,0,10000698
-0,2013-06-27,ABIN,63861,0,-99,5.4086,0,10000810
-0,2012-04-28,FP-GTC,15115,0,15,91.7724,0,10000414
-0,2013-04-19,ABIN,73506,0,13,14.8212,0,10000728
-0,2012-01-25,Others,32876,0,13,23.4295,0,10000673
-0,2013-11-06,ABIN,139973,3,12,41.9681,0,10000902
-0,2013-03-09,Auction,61323,0,15,53.8594,0,10000323
-0,2012-03-01,ABIN,60340,0,11,78.1279,0,10000947
-0,2012-07-28,FP-non GTC,279,15,16,80.8961,0,10000691
-0,2013-02-04,Others,24541,0,11,40.4673,0,10000727
-0,2012-02-24,Auction,152801,0,13,9.8921,0,10000661
-0,2013-08-13,ABIN,156614,0,15,61.6452,0,10000734
-0,2012-04-02,ABIN,63861,3,11,18.4644,0,10000446
-0,2013-09-28,Others,95173,0,-99,51.2096,0,10000049
-0,2013-08-13,FP-non GTC,64076,0,12,85.3952,0,10000581
-0,2012-09-09,ABIN,20485,0,15,0.5979,0,10000081
-0,2012-04-26,Auction,145970,0,11,40.5874,0,10000401
-0,2013-05-10,Others,26262,0,11,83.8561,0,10000545
-0,2013-05-23,FP-GTC,80287,0,11,27.6123,0,10000529
-0,2012-01-08,Others,175750,0,14,45.4268,0,10000408
-0,2012-07-20,ABIN,38238,0,16,44.1376,0,10000796
-0,2012-09-15,Auction,65,0,-99,97.3146,0,10000792
-0,2012-10-17,ABIN,65,0,12,68.5363,0,10000569
-0,2013-08-03,FP-GTC,40059,3,15,87.3892,0,10000317
-0,2013-08-28,FP-non GTC,80287,0,14,61.9829,0,10000015
-0,2012-07-21,Auction,25147,0,11,24.7662,0,10000333
-0,2013-10-17,Auction,132939,0,11,11.6038,0,10000926
-0,2012-09-04,Auction,44079,0,15,75.1095,0,10000435
-0,2012-11-01,Auction,67698,0,5,8.8273,0,10000834
-0,2013-12-02,Auction,24541,0,15,95.2195,0,10000259
-0,2012-02-06,Auction,533,0,12,66.4962,0,10000007
-0,2013-12-03,FP-non GTC,99985,0,12,0.0537,0,10000244
-0,2013-05-11,FP-non GTC,60606,3,12,76.6828,0,10000002
-0,2012-09-18,Auction,156614,0,16,72.1912,0,10000174
-0,2012-05-19,Auction,60340,0,16,42.9975,0,10000813
-0,2012-04-26,FP-GTC,165888,0,14,88.3875,0,10000919
-0,2013-08-30,Others,963,0,15,6.6424,0,10000994
-0,2013-08-28,ABIN,10866,0,12,79.7295,0,10000832
-0,2012-01-16,ABIN,145970,0,11,72.3940,0,10000008
-0,2013-12-23,Others,65,0,15,94.0397,0,10000812
-0,2013-09-06,FP-GTC,25147,0,11,72.9395,0,10000956
-0,2012-05-07,FP-GTC,16509,0,11,9.8290,0,10000940
-0,2013-09-07,Others,139973,0,11,22.9964,0,10000563
-0,2012-11-17,Others,65,0,-99,62.7316,0,10000227
-0,2012-05-14,ABIN,57013,0,13,19.0969,0,10000844
-0,2012-06-01,FP-non GTC,30059,3,16,26.9105,0,10000516
-0,2012-11-24,Auction,75665,0,15,26.5788,0,10000717
-0,2013-10-23,FP-non GTC,75708,3,12,10.5298,0,10000238
-0,2012-10-19,Auction,64076,0,11,49.0746,0,10000389
-0,2012-02-17,ABIN,65,0,12,58.3004,0,10000219
-0,2012-01-08,FP-GTC,15115,0,16,3.2380,0,10000435
-0,2012-02-04,Others,164261,0,14,45.9304,0,10000748
-0,2012-08-25,FP-GTC,156356,0,5,70.3654,0,10000864
-0,2013-04-10,FP-GTC,38238,0,5,72.9872,0,10000638
-0,2013-05-16,FP-GTC,175750,0,5,95.3008,0,10000366
-0,2012-06-27,FP-GTC,100847,0,14,64.8447,0,10000686
-0,2013-03-06,Auction,139255,15,12,20.7811,0,10000261
-0,2013-08-04,Auction,67698,0,14,20.1344,0,10000142
-0,2013-08-15,Auction,20865,0,12,64.6842,0,10000235
-0,2012-04-17,Auction,24760,0,-99,88.4159,0,10000039
-0,2013-03-14,Others,75665,0,13,36.9770,0,10000480
-0,2012-12-09,Others,139973,0,15,85.1056,0,10000801
-0,2012-07-02,Others,36250,0,11,71.5548,0,10000926
-0,2013-08-04,FP-non GTC,31519,0,16,23.9847,0,10000680
-0,2013-02-15,FP-non GTC,80053,0,15,68.4438,0,10000487
-0,2012-07-12,FP-non GTC,64076,0,5,54.5740,0,10000628
-0,2012-11-07,Others,164,0,12,45.1637,0,10000846
-0,2013-09-14,FP-GTC,106340,15,15,38.1572,0,10000201
-0,2013-12-19,Auction,759,0,-99,53.0086,0,10000250
-0,2012-05-11,Others,16509,0,-99,68.3043,0,10000809
-0,2013-07-20,Others,45333,0,15,58.5554,0,10000981
-0,2013-03-24,FP-GTC,20485,0,5,62.1012,0,10000877
-0,2013-08-09,ABIN,63861,0,5,44.1975,0,10000554
-0,2013-12-04,FP-GTC,53064,0,14,29.8358,0,10000491
-0,2013-07-06,ABIN,139973,0,5,26.8986,0,10000978
-0,2013-04-29,FP-non GTC,67698,0,11,46.9203,0,10000148
-0,2013-10-18,FP-non GTC,95173,0,14,25.5313,0,10000068
-0,2014-01-01,Auction,6762,0,13,30.6746,0,10000592
-0,2013-07-14,Auction,13836,0,16,79.4600,0,10000944
-0,2012-10-12,FP-GTC,145970,0,5,32.3728,0,10000396
-0,2013-07-28,Auction,1357,0,13,8.7896,0,10000836
-0,2013-12-06,FP-non GTC,73506,0,-99,95.4405,0,10000914
-0,2012-08-01,FP-non GTC,37831,0,11,12.8673,0,10000993
-0,2013-09-07,FP-non GTC,223,0,-99,16.9111,0,10000028
-0,2012-12-27,FP-GTC,82494,15,15,73.5475,0,10000609
-0,2012-04-07,Others,36250,0,16,7.5886,0,10000838
-0,2012-10-07,FP-GTC,139973,0,12,45.3171,0,10000439
-0,2012-01-19,FP-non GTC,63861,0,16,31.1155,0,10000338
-0,2013-12-16,FP-non GTC,164,0,16,41.0688,0,10000377
-0,2012-03-25,ABIN,41940,0,14,8.1251,0,10000318
-0,2012-04-15,FP-non GTC,65,0,-99,56.9993,0,10000697
-0,2012-11-27,Auction,25147,0,14,14.5707,0,10000303
-0,2012-08-04,FP-non GTC,20213,0,11,20.6361,0,10000443
-0,2013-11-15,Auction,95672,3,5,40.1888,0,10000506
-0,2013-12-26,Others,16145,0,13,34.4427,0,10000307
-0,2013-11-25,Others,139973,0,16,60.8826,0,10000762
-0,2013-09-01,Others,24760,0,16,22.9159,0,10000759
-0,2012-12-10,ABIN,53064,0,14,51.2661,0,10000626
-0,2013-08-24,Auction,759,0,13,48.4301,0,10000370
-0,2012-11-29,Auction,32876,0,5,86.2750,0,10000755
-0,2012-06-11,Others,61323,0,13,14.7483,0,10000812
-0,2012-04-23,Others,67698,0,-99,21.2161,0,10000384
-0,2013-07-12,Others,40059,3,5,88.2983,0,10000937
-0,2012-12-27,FP-GTC,13987,0,5,29.1748,0,10000223
-0,2013-02-15,Others,113802,15,5,31.4243,0,10000672
-0,2013-02-20,Auction,166013,15,15,22.9952,0,10000618
-0,2013-03-13,ABIN,73506,0,16,94.7440,0,10000476
-0,2012-02-19,FP-GTC,95672,0,15,28.6221,0,10000421
-0,2013-11-30,FP-GTC,166013,15,5,92.5521,0,10000729
-0,2012-09-30,ABIN,36250,0,15,18.6346,0,10000606
-0,2013-02-13,Auction,20886,0,14,24.7987,0,10000535
-0,2013-11-10,FP-GTC,759,0,13,53.7935,0,10000711
-0,2013-02-10,Auction,20886,0,12,84.2953,0,10000811
-0,2012-05-28,ABIN,10866,0,13,64.3564,0,10000804
-0,2013-03-19,Auction,314,0,16,36.1791,0,10000379
-0,2012-01-26,FP-non GTC,132939,0,-99,31.4514,0,10000690
-0,2013-02-02,FP-non GTC,20485,0,14,97.4265,0,10000642
-0,2013-07-25,Auction,139973,0,16,12.3287,0,10000972
-0,2012-09-21,Others,15115,0,11,2.7636,0,10000896
-0,2012-07-26,ABIN,533,0,15,0.6745,0,10000762
-0,2012-06-02,FP-non GTC,48904,0,-99,56.1505,0,10000937
-0,2012-12-11,FP-non GTC,155226,0,14,59.4547,0,10000197
-0,2012-05-30,Others,26249,0,14,18.0992,0,10000836
-0,2012-12-25,Auction,20485,0,15,23.8257,0,10000992
-0,2013-07-08,ABIN,57013,0,12,59.6550,0,10000196
-0,2013-02-22,Auction,156356,0,14,36.1970,0,10000014
-0,2012-03-28,ABIN,25147,0,-99,56.3283,0,10000943
-0,2012-10-09,Others,63861,0,16,9.5503,0,10000659
-0,2012-12-29,FP-non GTC,95672,0,12,57.7305,0,10000568
-0,2013-06-14,Others,80053,0,13,89.8029,0,10000580
-0,2012-02-14,Others,67698,0,16,73.0734,0,10000857
-0,2013-05-14,FP-non GTC,62179,0,5,58.7369,0,10000091
-0,2013-06-16,FP-GTC,1504,0,5,7.2913,0,10000054
-0,2013-07-30,FP-non GTC,31673,0,-99,13.2961,0,10000654
-0,2013-05-22,ABIN,130,0,-99,1.3005,0,10000925
-0,2012-03-06,FP-GTC,35570,100,15,45.7240,0,10000773
-0,2012-07-20,Auction,15868,0,15,88.4248,0,10000965
-0,2013-10-02,Auction,38238,0,15,34.7237,0,10000987
-0,2012-11-06,Others,67698,0,-99,3.3750,0,10000902
-0,2012-01-11,FP-GTC,62179,0,11,77.9520,0,10000540
-0,2012-09-23,FP-GTC,64076,0,12,57.1896,0,10000772
-0,2012-03-03,ABIN,103178,15,11,26.3886,0,10000000
-0,2012-05-26,FP-GTC,48027,0,16,87.2722,0,10000836
-0,2012-05-06,ABIN,38238,0,16,34.7669,0,10000942
-0,2014-01-01,Auction,170302,15,11,62.2724,0,10000988
-0,2012-06-19,FP-GTC,37831,0,13,78.1533,0,10000362
-0,2012-01-12,ABIN,48904,0,-99,80.2518,0,10000266
-0,2013-06-24,ABIN,10866,0,12,87.3794,0,10000655
-0,2013-07-17,Others,1349,0,12,40.6979,0,10000754
-0,2013-06-14,Auction,139973,0,14,54.9050,0,10000105
-0,2012-06-18,ABIN,15868,0,-99,69.2751,0,10000988
-0,2012-09-07,Auction,1349,0,11,24.7687,0,10000323
-0,2012-02-01,ABIN,61323,0,-99,72.0754,0,10000321
-0,2013-07-28,ABIN,152801,0,11,71.8684,0,10000701
-0,2013-04-16,Others,60340,0,-99,9.8364,0,10000952
-0,2012-01-06,ABIN,80053,0,12,45.9154,0,10000726
-0,2012-02-07,Auction,223,0,12,90.7522,0,10000949
-0,2012-09-15,Others,50677,0,5,50.2863,0,10000285
-0,2013-04-01,Auction,75665,0,12,40.7265,0,10000498
-0,2012-11-09,ABIN,95672,0,12,85.7492,0,10000180
-0,2013-09-22,FP-GTC,16145,0,16,37.4930,0,10000453
-0,2012-11-02,Auction,73506,0,5,61.7567,0,10000361
-0,2013-10-27,Auction,963,0,-99,26.4725,0,10000909
-0,2012-01-13,FP-GTC,10866,0,16,84.3734,0,10000539
-0,2012-05-26,FP-non GTC,152801,0,5,77.1831,0,10000961
-0,2013-06-04,ABIN,57013,0,-99,59.7825,0,10000542
-0,2012-11-26,FP-GTC,9426,3,5,83.8080,0,10000847
-0,2013-12-11,ABIN,67698,0,14,67.3561,0,10000409
-0,2013-06-01,ABIN,95173,0,13,50.4721,0,10000245
-0,2012-07-25,FP-GTC,175750,0,16,62.5123,0,10000399
-0,2012-01-22,FP-GTC,31673,0,13,8.1462,0,10000125
-0,2012-01-17,Auction,63861,0,15,86.7921,0,10000067
-0,2012-07-06,Others,20886,0,-99,91.8578,0,10000349
-0,2012-02-26,FP-non GTC,216,0,13,25.5255,0,10000465
-0,2013-10-05,Others,65,0,11,50.8591,0,10000718
-0,2012-12-30,FP-non GTC,87118,0,11,81.3002,0,10000122
-0,2013-08-20,ABIN,130,0,13,61.5051,0,10000685
-0,2012-11-07,ABIN,57013,0,15,87.1409,0,10000157
-0,2012-11-26,Others,75665,0,16,85.4010,0,10000935
-0,2013-12-17,FP-GTC,63861,0,14,45.7096,0,10000650
-0,2012-08-12,FP-non GTC,20485,0,-99,44.4235,0,10000954
-0,2012-01-17,Auction,314,0,12,75.8335,0,10000514
-0,2013-01-24,ABIN,2635,0,12,10.6521,0,10000839
-0,2012-02-05,ABIN,3838,0,12,76.2544,0,10000344
-0,2013-05-13,FP-GTC,106246,0,14,25.5924,0,10000291
-0,2013-03-16,Auction,82494,15,5,15.6450,0,10000366
-0,2012-01-06,Auction,65,0,11,9.8382,0,10000415
-0,2012-08-10,ABIN,24541,0,-99,21.1252,0,10000872
-0,2012-06-01,Others,106246,0,11,51.6742,0,10000000
-0,2013-05-10,Others,63861,0,-99,69.1514,0,10000869
-0,2013-04-04,FP-non GTC,73506,0,13,7.8706,0,10000986
-0,2013-10-23,ABIN,43972,100,-99,31.5134,0,10000011
-0,2013-05-10,Auction,80053,0,-99,2.8786,0,10000544
-0,2012-03-26,FP-non GTC,80053,0,13,59.3087,0,10000881
-0,2012-07-24,Auction,20485,0,-99,49.9084,0,10000757
-0,2013-04-10,ABIN,66767,15,5,48.4303,0,10000471
-0,2013-07-09,ABIN,13987,0,12,82.6598,0,10000495
-0,2012-12-25,ABIN,145970,0,5,6.4088,0,10000723
-0,2013-11-14,FP-non GTC,113593,15,15,10.9465,0,10000701
-0,2013-08-17,Auction,26249,0,12,90.2925,0,10000684
-0,2013-03-28,Auction,53064,0,13,1.5558,0,10000884
-0,2013-05-04,ABIN,88750,0,15,41.8885,0,10000078
-0,2013-07-20,Others,60340,0,-99,79.8564,0,10000590
-0,2012-12-10,ABIN,1504,0,13,22.2111,0,10000603
-0,2012-01-18,FP-non GTC,1349,0,-99,40.5209,0,10000684
-0,2013-01-21,FP-non GTC,25147,0,13,47.2094,0,10000676
-0,2013-10-18,FP-non GTC,164261,0,5,71.2879,0,10000998
-0,2012-08-18,Others,61323,0,15,59.1738,0,10000705
-0,2012-01-28,ABIN,1357,0,11,31.3101,0,10000131
-0,2013-12-25,Others,94847,0,13,90.8027,0,10000674
-0,2013-12-26,Others,156614,0,5,14.9093,0,10000123
-0,2012-06-19,Others,43398,0,13,85.5022,0,10000874
-0,2012-12-12,FP-non GTC,145970,0,16,71.8714,0,10000686
-0,2012-02-11,Others,24760,0,14,49.0051,0,10000611
-0,2013-06-02,Others,759,0,13,56.3362,0,10000185
-0,2012-09-30,Auction,75665,0,16,89.9688,0,10000378
-0,2013-11-10,Others,43479,0,16,2.0896,0,10000272
-0,2012-06-19,FP-GTC,759,0,5,34.9300,0,10000531
-0,2012-02-02,FP-non GTC,57990,3,-99,12.9845,0,10000591
-0,2013-04-04,FP-GTC,73506,0,11,32.9647,0,10000464
-0,2012-07-01,FP-non GTC,24760,0,11,84.6443,0,10000646
-0,2012-10-29,ABIN,46575,0,5,71.8566,0,10000247
-0,2013-11-19,Others,25147,0,16,61.2794,0,10000146
-0,2013-09-10,Others,95672,0,13,13.8226,0,10000353
-0,2013-07-29,FP-non GTC,67698,0,16,36.8872,0,10000266
-0,2013-10-17,ABIN,73506,0,11,57.0211,0,10000341
-0,2013-03-25,Others,20213,0,11,55.2006,0,10000627
-0,2012-05-30,FP-GTC,38238,0,5,9.9162,0,10000418
-0,2013-07-17,Auction,100847,0,11,11.3573,0,10000097
-0,2012-02-11,Others,166013,15,16,49.2212,0,10000634
-0,2012-08-17,FP-GTC,45333,0,16,99.5103,0,10000769
-0,2012-03-26,Auction,11554,0,14,58.2225,0,10000444
-0,2012-11-16,FP-non GTC,100847,0,16,67.4826,0,10000523
-0,2012-06-22,Auction,20213,0,5,44.8452,0,10000639
-0,2012-07-04,ABIN,1504,0,16,64.0222,0,10000827
-0,2012-10-10,Others,16145,0,14,81.1907,0,10000468
-0,2012-06-12,FP-non GTC,45333,0,14,20.5268,0,10000808
-0,2012-07-27,FP-non GTC,156614,0,11,28.1814,0,10000053
-0,2012-05-01,ABIN,61323,0,12,78.4665,0,10000767
-0,2013-05-04,ABIN,20485,0,16,94.0624,0,10000715
-0,2013-06-21,ABIN,20485,0,-99,92.7954,0,10000066
-0,2013-09-23,FP-non GTC,25147,0,5,81.3970,0,10000728
-0,2013-01-13,Auction,88750,0,-99,18.9880,0,10000265
-0,2012-01-12,FP-GTC,2023,0,14,91.3572,0,10000139
-0,2013-11-08,Auction,63861,0,11,17.1466,0,10000247
-0,2012-12-02,Auction,87118,0,5,75.0804,0,10000364
-0,2013-09-23,Others,46575,0,11,10.2322,0,10000384
-0,2012-01-26,FP-GTC,20865,0,16,6.7215,0,10000538
-0,2013-09-11,Others,158666,15,14,10.9434,0,10000153
-0,2013-02-19,Others,106340,15,16,32.2718,0,10000481
-0,2012-08-21,FP-non GTC,57784,0,11,29.4936,0,10000682
-0,2013-12-17,ABIN,314,0,13,45.6416,0,10000323
-0,2012-11-28,ABIN,46575,0,11,74.3051,0,10000167
-0,2013-10-25,Auction,1357,0,16,63.2572,0,10000556
-0,2012-08-13,ABIN,51582,0,16,12.5932,0,10000744
-0,2012-03-14,FP-GTC,1161,3,15,36.8516,0,10000221
-0,2012-10-12,Others,1349,0,13,9.8277,0,10000954
-0,2013-07-30,Auction,16145,0,13,25.6803,0,10000527
-0,2013-04-20,Auction,121153,0,-99,94.4016,0,10000033
-0,2013-12-11,FP-GTC,45333,0,13,50.3231,0,10000488
-0,2012-03-12,Others,80135,0,14,46.9661,0,10000800
-0,2012-01-26,FP-non GTC,62179,0,15,6.7796,0,10000298
-0,2012-07-18,Others,13836,0,12,59.3458,0,10000932
-0,2013-11-01,Others,13836,0,13,7.5394,0,10000014
-0,2013-06-11,Auction,16509,0,12,53.2002,0,10000309
-0,2012-01-14,Auction,20865,0,12,9.5903,0,10000003
-0,2013-12-13,FP-GTC,43479,0,5,28.2234,0,10000654
-0,2012-02-17,Auction,6762,0,5,84.1040,0,10000246
-0,2013-11-01,FP-GTC,16145,0,5,44.8790,0,10000358
-0,2013-08-26,Others,67698,0,5,24.7653,0,10000110
-0,2012-02-01,FP-non GTC,99985,0,5,73.9172,0,10000822
-0,2013-01-05,Auction,11554,0,13,82.7057,0,10000554
-0,2012-12-05,FP-GTC,13987,0,-99,91.9448,0,10000377
-0,2012-04-27,FP-non GTC,13836,0,14,93.1923,0,10000766
-0,2012-09-23,ABIN,43479,0,16,74.2753,0,10000603
-0,2013-11-21,ABIN,164,0,14,81.0875,0,10000489
-0,2013-04-04,Auction,170083,3,12,12.4107,0,10000183
-0,2013-02-02,Auction,30059,3,11,66.0915,0,10000644
-0,2013-09-29,Auction,2635,0,13,41.5935,0,10000488
-0,2012-11-27,ABIN,26262,0,16,27.6081,0,10000191
-0,2013-01-10,Others,130,0,11,19.0647,0,10000056
-0,2013-07-28,FP-non GTC,164261,0,5,25.9751,0,10000663
-0,2012-10-05,FP-GTC,10866,0,-99,42.0352,0,10000058
-0,2013-11-22,Auction,20485,0,16,18.9452,0,10000996
-0,2013-11-30,FP-non GTC,175750,3,11,10.0374,0,10000358
-0,2013-04-23,ABIN,113802,15,15,40.0768,0,10000497
-0,2012-09-17,FP-GTC,216,0,14,32.8424,0,10000113
-0,2012-11-02,FP-non GTC,16145,3,14,63.9673,0,10000471
-0,2013-06-27,ABIN,67698,0,16,35.2993,0,10000722
-0,2013-12-16,ABIN,121153,0,-99,60.6539,0,10000709
-0,2012-03-27,ABIN,20886,0,12,69.1836,0,10000768
-0,2013-11-04,ABIN,57013,0,14,37.7805,0,10000450
-0,2012-09-04,FP-non GTC,57990,0,13,13.8693,0,10000743
-0,2013-06-22,Auction,20485,0,16,17.1755,0,10000336
-0,2012-10-14,FP-GTC,156614,0,12,29.0308,0,10000577
-0,2013-08-04,ABIN,106246,0,13,98.4005,0,10000297
-0,2012-06-13,ABIN,43398,0,16,95.3892,0,10000984
-0,2012-05-21,Auction,20485,0,12,44.0443,0,10000357
-0,2013-08-12,Auction,165888,0,14,0.5521,0,10000804
-0,2013-03-12,Others,95672,0,14,54.3816,0,10000338
-0,2012-11-29,Others,15868,0,11,9.4996,0,10000177
-0,2013-10-25,FP-GTC,31519,0,12,58.8241,0,10000720
-0,2012-03-08,FP-GTC,165888,0,16,46.6778,0,10000484
-0,2012-04-30,Auction,1504,0,11,48.3609,0,10000835
-0,2012-08-31,FP-GTC,314,0,15,41.9195,0,10000284
-0,2012-04-02,Auction,33038,15,13,89.7577,0,10000947
-0,2013-07-21,Auction,45333,0,16,0.4349,0,10000941
-0,2012-07-13,ABIN,16509,0,5,98.1480,0,10000363
-0,2013-06-05,ABIN,4943,0,5,96.3653,0,10000029
-0,2013-10-12,FP-GTC,15115,0,12,64.1499,0,10000339
-0,2012-11-08,FP-GTC,25147,0,15,87.9610,0,10000149
-0,2013-02-17,Others,61323,0,16,92.5311,0,10000932
-0,2012-04-26,Others,20485,0,11,20.2247,0,10000245
-0,2013-07-13,FP-non GTC,95173,0,16,95.5853,0,10000150
-0,2012-10-08,FP-GTC,11554,0,15,29.6721,0,10000584
-0,2012-01-05,ABIN,31519,0,5,40.7845,0,10000365
-0,2013-07-11,ABIN,139973,0,16,64.1171,0,10000712
-0,2012-02-24,FP-GTC,62179,0,-99,93.5801,0,10000634
-0,2012-07-29,Others,152801,0,-99,23.4592,0,10000351
-0,2012-08-19,Others,10866,0,5,90.3864,0,10000665
-0,2012-05-10,ABIN,174053,3,14,90.5710,0,10000685
-0,2013-11-10,Auction,46575,0,-99,86.3150,0,10000982
-0,2012-05-03,FP-GTC,67703,3,14,17.5203,0,10000985
-0,2013-04-13,Auction,20213,0,-99,28.1159,0,10000357
-0,2013-12-01,FP-non GTC,13987,0,5,75.0139,0,10000703
-0,2013-06-06,Auction,10866,0,5,57.2098,0,10000916
-0,2013-04-20,Others,63861,0,15,97.3023,0,10000021
-0,2012-02-01,FP-non GTC,13836,0,13,54.2758,0,10000217
-0,2013-11-01,FP-non GTC,99985,0,12,76.3203,0,10000817
-0,2013-02-23,Others,80287,0,13,17.8778,0,10000937
-0,2012-04-22,Auction,3838,0,14,64.5441,0,10000466
-0,2013-03-14,Others,20485,0,11,25.2019,0,10000098
-0,2012-04-01,Others,31519,3,15,65.6555,0,10000121
-0,2012-01-28,FP-GTC,60606,3,14,11.4888,0,10000057
-0,2012-12-23,Auction,31519,0,14,38.8159,0,10000563
-0,2013-02-12,ABIN,4943,0,14,99.1697,0,10000525
-0,2013-08-07,Auction,67698,0,-99,42.5442,0,10000547
-0,2012-12-21,FP-non GTC,57990,0,11,56.5816,0,10000092
-0,2013-02-28,FP-GTC,20485,0,14,13.4842,0,10000463
-0,2013-09-26,ABIN,10866,0,16,28.3103,0,10000185
-0,2013-05-16,FP-GTC,63889,0,-99,16.0623,0,10000533
-0,2012-04-03,ABIN,32876,0,11,88.8682,0,10000702
-0,2013-02-28,FP-GTC,50508,0,-99,80.6989,0,10000669
-0,2012-07-24,ABIN,95672,0,12,77.5250,0,10000322
-0,2013-07-21,Auction,31519,0,11,26.2578,0,10000442
-0,2012-07-18,Others,43398,0,11,16.4674,0,10000831
-0,2013-07-31,Auction,26262,0,11,60.5816,0,10000182
-0,2013-12-18,ABIN,164261,0,14,61.6805,0,10000522
-0,2013-03-31,FP-GTC,25147,0,13,75.9632,0,10000685
-0,2012-05-30,FP-GTC,15687,0,13,61.0666,0,10000600
-0,2012-04-15,FP-non GTC,57784,0,5,30.0380,0,10000419
-0,2013-05-19,ABIN,139973,3,15,15.0669,0,10000388
-0,2013-02-20,Others,164261,0,15,18.6774,0,10000276
-0,2012-03-16,FP-non GTC,61323,0,14,49.2798,0,10000187
-0,2013-11-22,FP-GTC,62179,0,-99,92.8435,0,10000014
-0,2013-08-25,ABIN,1504,0,-99,39.5945,0,10000183
-0,2013-08-14,FP-non GTC,1357,0,-99,47.7304,0,10000079
-0,2012-12-19,Others,32876,0,16,67.1409,0,10000867
-0,2012-03-28,Others,80135,0,16,62.6308,0,10000577
-0,2013-01-01,FP-GTC,38238,0,5,52.0108,0,10000514
-0,2012-02-28,Auction,63889,0,15,28.0304,0,10000595
-0,2013-08-04,Auction,61323,0,16,38.9078,0,10000608
-0,2012-06-21,FP-GTC,95173,0,15,94.9831,0,10000156
-0,2013-06-06,Auction,57013,0,11,75.3949,0,10000992
-0,2012-08-08,FP-non GTC,26249,0,11,64.1580,0,10000469
-0,2012-05-13,ABIN,132939,0,-99,71.6430,0,10000542
-0,2012-03-06,FP-non GTC,57990,0,11,77.1134,0,10000093
-0,2012-08-07,ABIN,16145,3,16,65.5825,0,10000841
-0,2012-07-29,FP-GTC,164,0,5,62.5545,0,10000552
-0,2012-07-26,FP-non GTC,37831,0,13,2.1842,0,10000802
-0,2013-11-23,ABIN,13987,0,11,9.7841,0,10000094
-0,2013-12-13,Auction,1349,0,11,93.3393,0,10000024
-0,2013-09-25,ABIN,80053,0,13,0.2014,0,10000693
-0,2012-05-17,ABIN,31519,0,16,39.1773,0,10000060
-0,2013-02-27,ABIN,175750,0,15,44.1286,0,10000721
-0,2012-04-04,Others,1349,0,16,45.4831,0,10000828
-0,2012-04-12,Auction,63889,0,-99,84.5028,0,10000402
-0,2013-07-02,FP-non GTC,80053,0,5,44.1441,0,10000742
-0,2013-12-22,Others,1349,0,13,83.7579,0,10000223
-0,2013-06-10,ABIN,95173,0,15,20.0108,0,10000952
-0,2012-10-20,Auction,11848,0,13,33.2825,0,10000329
-0,2013-10-17,ABIN,26249,0,16,18.9369,0,10000685
-0,2012-08-20,FP-non GTC,3838,0,-99,35.9469,0,10000069
-0,2012-02-12,FP-GTC,31519,3,14,35.2504,0,10000991
-0,2012-01-09,FP-GTC,100847,0,13,99.8144,0,10000782
-0,2013-09-09,Auction,73506,0,16,36.6553,0,10000835
-0,2013-07-30,FP-non GTC,67703,3,16,3.5994,0,10000203
-0,2012-05-09,Others,139973,0,13,8.8491,0,10000074
-0,2012-10-13,FP-GTC,20865,0,14,19.0378,0,10000747
-0,2012-09-10,FP-GTC,158798,0,11,37.8799,0,10000312
-0,2012-07-25,FP-GTC,95173,0,14,83.3301,0,10000893
-0,2013-08-19,Auction,166013,15,16,41.9336,0,10000846
-0,2012-02-13,Others,87118,0,12,93.4500,0,10000136
-0,2013-07-06,FP-non GTC,1120,3,-99,45.6255,0,10000086
-0,2012-12-20,Auction,50677,0,12,15.1782,0,10000452
-0,2013-08-20,FP-non GTC,165888,0,11,63.8546,0,10000780
-0,2012-11-29,FP-GTC,63861,0,14,35.9757,0,10000134
-0,2013-09-10,Auction,95672,0,-99,13.9615,0,10000553
-0,2013-12-11,Others,314,0,14,61.2280,0,10000269
-0,2013-05-27,FP-GTC,1504,0,16,89.8299,0,10000168
-0,2012-06-22,FP-non GTC,94847,0,11,86.5431,0,10000641
-0,2012-02-24,FP-non GTC,314,0,14,7.0223,0,10000288
-0,2013-01-12,FP-GTC,20865,0,15,99.4581,0,10000069
-0,2013-10-28,Auction,174106,3,15,19.0917,0,10000198
-0,2012-01-29,Auction,121153,0,11,16.5034,0,10000492
-0,2012-12-02,FP-non GTC,53064,0,5,40.0250,0,10000002
-0,2013-04-17,ABIN,60340,0,13,96.5255,0,10000357
-0,2013-12-16,Auction,61323,0,15,72.7839,0,10000195
-0,2012-06-20,Auction,37831,0,5,8.3551,0,10000429
-0,2012-11-22,ABIN,31387,3,14,17.2673,0,10000854
-0,2012-11-22,FP-GTC,95173,0,-99,70.2540,0,10000619
-0,2013-09-30,FP-non GTC,57990,0,11,81.0956,0,10000431
-0,2013-08-09,ABIN,155226,0,-99,41.4198,0,10000204
-0,2012-09-03,Auction,88750,0,11,45.2273,0,10000947
-0,2012-01-01,FP-non GTC,156356,0,12,44.7485,0,10000439
-0,2012-04-21,ABIN,50677,0,16,80.4618,0,10000466
-0,2013-03-30,Others,20213,0,12,86.3777,0,10000781
-0,2012-05-18,FP-GTC,12688,0,14,11.7047,0,10000550
-0,2013-08-18,Auction,31519,0,13,57.6148,0,10000608
-0,2013-10-27,FP-non GTC,24760,0,5,34.9687,0,10000322
-0,2013-03-26,FP-GTC,1349,0,11,51.2759,0,10000800
-0,2012-08-31,FP-GTC,4943,0,14,43.3525,0,10000916
-0,2013-10-27,FP-GTC,99985,0,13,91.0379,0,10000091
-0,2012-07-10,Others,24760,0,16,39.2775,0,10000813
-0,2012-11-06,FP-non GTC,50508,0,16,22.9019,0,10000271
-0,2013-11-27,FP-non GTC,175750,0,16,36.9125,0,10000898
-0,2013-12-22,FP-GTC,80053,0,13,83.0584,0,10000443
-0,2012-12-16,Auction,6762,0,14,97.8880,0,10000723
-0,2012-08-08,Others,57990,0,5,65.6268,0,10000715
-0,2012-07-23,FP-GTC,48027,0,-99,98.0398,0,10000565
-0,2012-12-20,FP-GTC,106246,0,12,67.1574,0,10000293
-0,2013-11-01,ABIN,1349,0,5,10.9577,0,10000369
-0,2012-04-28,Auction,139973,3,11,25.6617,0,10000067
-0,2012-11-01,FP-non GTC,155226,0,14,65.7990,0,10000083
-0,2013-05-27,FP-GTC,48027,0,13,61.6990,0,10000100
-0,2013-08-07,FP-GTC,314,0,14,30.0501,0,10000344
-0,2013-03-13,FP-non GTC,175750,0,-99,82.5221,0,10000683
-0,2013-07-30,FP-GTC,31519,3,16,95.1425,0,10000606
-0,2013-12-18,Auction,50508,0,15,93.7747,0,10000283
-0,2012-11-28,FP-non GTC,53064,0,11,65.5664,0,10000903
-0,2013-10-04,Others,26249,0,14,66.2702,0,10000149
-0,2012-02-01,ABIN,15687,0,-99,55.4663,0,10000025
-0,2012-11-09,Auction,132939,0,5,0.4841,0,10000477
-0,2012-05-31,FP-non GTC,67698,0,13,5.6112,0,10000205
-0,2012-01-10,ABIN,170083,3,14,91.3684,0,10000968
-0,2013-10-02,FP-GTC,41940,0,14,85.7042,0,10000503
-0,2013-06-20,Others,106340,15,11,64.5684,0,10000573
-0,2013-03-30,FP-GTC,87118,0,12,11.5656,0,10000456
-0,2013-05-20,Others,37831,0,16,76.5619,0,10000829
-0,2012-01-30,Others,16145,0,16,0.1008,0,10000079
-0,2013-12-08,FP-GTC,46575,0,14,44.8329,0,10000439
-0,2013-01-14,ABIN,73506,0,-99,14.5712,0,10000568
-0,2012-04-19,Others,130,0,13,73.6383,0,10000907
-0,2012-11-09,Others,11848,0,15,6.9025,0,10000095
-0,2013-03-15,FP-GTC,175750,0,5,55.2025,0,10000359
-0,2012-02-18,FP-non GTC,2635,0,-99,81.9584,0,10000506
-0,2013-12-07,Auction,80135,0,12,85.1453,0,10000262
-0,2013-04-19,ABIN,174106,3,11,94.1872,0,10000252
-0,2012-09-04,FP-non GTC,6762,0,16,79.4180,0,10000819
-0,2012-11-21,Others,37831,0,15,72.2244,0,10000911
-0,2013-06-03,Auction,164261,0,16,3.6716,0,10000112
-0,2013-01-23,FP-non GTC,121153,0,11,85.4725,0,10000843
-0,2013-12-30,Auction,3838,0,5,28.5619,0,10000995
-0,2013-09-26,ABIN,95672,0,15,21.3948,0,10000679
-0,2013-06-07,FP-non GTC,10866,0,-99,92.6345,0,10000466
-0,2013-11-04,FP-non GTC,63861,0,13,60.7962,0,10000174
-0,2012-07-14,Others,48027,0,16,46.3174,0,10000603
-0,2013-04-28,ABIN,87118,0,11,8.2679,0,10000590
-0,2013-07-22,ABIN,41940,0,13,76.5107,0,10000159
-0,2012-03-03,Auction,44079,0,11,95.7853,0,10000806
-0,2013-09-20,Others,20213,0,5,35.7134,0,10000922
-0,2012-09-13,Auction,62179,0,-99,80.7493,0,10000356
-0,2012-06-15,ABIN,314,0,13,66.0961,0,10000139
-0,2013-01-29,Others,24541,0,-99,66.4687,0,10000803
-0,2013-06-19,Others,1357,0,15,35.6912,0,10000723
-0,2012-11-15,Others,80135,0,11,44.1951,0,10000083
-0,2012-10-22,Others,67698,0,12,6.8778,0,10000535
-0,2013-09-09,FP-non GTC,50508,0,16,31.1656,0,10000160
-0,2013-10-22,Others,3838,0,5,98.1652,0,10000875
-0,2012-10-11,Others,1349,0,15,31.2521,0,10000663
-0,2013-08-30,FP-GTC,20485,0,-99,15.8748,0,10000988
-0,2013-04-10,Others,38238,0,15,29.3240,0,10000686
-0,2013-07-20,Auction,24760,0,-99,57.0828,0,10000657
-0,2012-08-04,FP-GTC,159184,0,12,51.4276,0,10000895
-0,2013-06-28,FP-non GTC,13836,0,-99,27.7177,0,10000897
-0,2013-05-06,FP-non GTC,43398,0,5,51.7179,0,10000862
-0,2013-02-01,ABIN,15868,0,5,46.3238,0,10000744
-0,2013-05-12,Auction,533,0,5,83.6965,0,10000200
-0,2012-04-26,Auction,4943,0,16,59.3741,0,10000918
-0,2012-03-12,Others,48027,0,14,76.2845,0,10000564
-0,2012-08-15,ABIN,32876,0,-99,72.1322,0,10000483
-0,2012-03-03,Others,43479,0,15,40.9293,0,10000937
-0,2013-01-20,Auction,38238,0,15,59.0009,0,10000040
-0,2012-03-17,Others,38238,0,12,59.9178,0,10000267
-0,2012-03-30,ABIN,50508,0,11,27.4960,0,10000970
-0,2012-04-22,ABIN,53064,0,-99,48.6321,0,10000750
-0,2012-05-29,Others,1504,0,11,98.6906,0,10000143
-0,2013-07-02,FP-non GTC,80135,0,11,40.2555,0,10000353
-0,2012-01-14,FP-GTC,6762,0,13,7.0226,0,10000466
-0,2012-11-19,Others,130,0,12,99.7620,0,10000731
-0,2012-06-23,Others,6762,0,5,63.1128,0,10000103
-0,2013-06-16,FP-non GTC,16145,3,12,64.2030,0,10000530
-0,2013-12-30,Auction,99985,0,5,23.4290,0,10000972
-0,2013-06-09,Others,46575,0,13,13.4803,0,10000745
-0,2013-12-10,Auction,103324,15,16,99.7648,0,10000786
-0,2012-09-28,ABIN,24760,0,15,11.3547,0,10000673
-0,2012-11-29,Others,158798,0,-99,82.8002,0,10000586
-0,2012-04-16,Others,175750,0,12,16.9311,0,10000751
-0,2012-03-16,Auction,46575,0,5,3.2964,0,10000589
-0,2012-01-24,ABIN,150265,15,14,64.6865,0,10000434
-0,2012-09-26,Others,38238,0,12,73.3389,0,10000438
-0,2012-01-30,FP-GTC,15868,0,-99,5.6161,0,10000671
-0,2013-05-25,FP-non GTC,38238,0,-99,93.9862,0,10000669
-0,2012-11-15,FP-non GTC,6762,0,14,19.2198,0,10000276
-0,2012-08-17,FP-non GTC,64076,0,14,88.5536,0,10000347
-0,2013-06-02,ABIN,67698,2,11,24.6629,0,10000487
-0,2012-02-14,FP-non GTC,175750,0,11,28.6018,0,10000292
-0,2013-03-29,Auction,314,0,12,14.1619,0,10000757
-0,2012-02-24,FP-GTC,12688,0,5,48.2185,0,10000829
-0,2013-06-20,FP-non GTC,67698,0,5,68.2468,0,10000640
-0,2013-02-12,FP-GTC,57784,0,13,80.4582,0,10000435
-0,2013-06-20,Auction,26249,0,-99,60.6520,0,10000950
-0,2013-04-25,Auction,152801,0,15,76.0514,0,10000612
-0,2012-09-15,FP-non GTC,32876,0,16,56.6683,0,10000146
-0,2012-05-03,Others,1357,0,-99,1.3769,0,10000268
-0,2012-01-14,Auction,9426,3,11,71.7337,0,10000443
-0,2012-08-12,FP-non GTC,139973,0,5,80.5935,0,10000932
-0,2012-11-27,Others,175750,0,5,54.8804,0,10000716
-0,2013-08-28,FP-non GTC,16145,3,-99,70.7167,0,10000244
-0,2012-09-23,Auction,20213,0,16,4.5554,0,10000608
-0,2013-10-15,ABIN,1504,0,5,38.8845,0,10000803
-0,2013-04-30,FP-non GTC,6762,0,14,47.3754,0,10000538
-0,2012-01-06,ABIN,15868,0,5,33.4885,0,10000847
-0,2012-04-30,ABIN,57990,0,5,63.8902,0,10000821
-0,2012-09-27,ABIN,38238,0,16,43.0875,0,10000524
-0,2012-06-30,FP-non GTC,152801,0,12,29.8433,0,10000102
-0,2013-05-25,ABIN,40059,3,5,43.0317,0,10000899
-0,2013-08-11,FP-non GTC,10866,0,15,50.1175,0,10000211
-0,2013-05-28,Auction,150265,15,14,79.0973,0,10000675
-0,2012-10-01,FP-non GTC,48904,0,13,28.9811,0,10000301
-0,2013-04-14,FP-GTC,103324,15,13,45.0386,0,10000853
-0,2012-07-23,Auction,20485,0,13,21.4452,0,10000116
-0,2013-01-11,ABIN,80287,0,5,90.5218,0,10000503
-0,2012-10-07,FP-GTC,25147,0,16,98.8443,0,10000667
-0,2012-12-12,FP-non GTC,100847,0,16,27.8064,0,10000075
-0,2012-04-24,ABIN,165888,0,-99,41.8494,0,10000323
-0,2012-01-13,FP-GTC,80053,0,14,30.7342,0,10000781
-0,2013-02-23,FP-GTC,121153,0,-99,50.0159,0,10000361
-0,2012-10-22,ABIN,1349,0,11,39.3281,0,10000951
-0,2012-01-20,ABIN,2635,0,-99,34.4591,0,10000433
-0,2012-04-14,FP-GTC,26262,0,13,34.2101,0,10000000
-0,2012-07-20,Others,12688,0,5,83.1926,0,10000755
-0,2012-09-12,FP-non GTC,216,0,11,61.5914,0,10000504
-0,2012-01-29,Others,13987,0,5,49.9596,0,10000340
-0,2013-01-10,ABIN,65,0,12,84.5331,0,10000988
-0,2012-08-20,FP-GTC,2023,0,16,46.2586,0,10000802
-0,2013-11-24,Others,156614,0,-99,68.9025,0,10000372
-0,2012-12-02,Auction,158798,0,16,62.5343,0,10000020
-0,2012-06-12,FP-non GTC,175750,0,14,21.6343,0,10000451
-0,2013-02-11,ABIN,9426,3,14,54.9943,0,10000615
-0,2012-11-01,FP-non GTC,88750,0,-99,16.7597,0,10000145
-0,2013-08-16,Auction,15868,0,15,14.8127,0,10000190
-0,2013-07-30,Auction,45333,0,14,67.5141,0,10000870
-0,2012-01-05,Others,24541,0,15,92.5069,0,10000821
-0,2013-04-30,Auction,57990,3,16,39.4144,0,10000140
-0,2013-01-03,Auction,314,0,14,73.2865,0,10000848
-0,2012-02-05,ABIN,38238,0,5,21.4669,0,10000158
-0,2012-05-08,FP-non GTC,13987,0,16,15.7925,0,10000816
-0,2012-12-08,Others,164261,0,12,33.2040,0,10000614
-0,2013-08-14,FP-non GTC,139255,15,16,37.1862,0,10000745
-0,2012-08-29,Auction,156614,0,14,11.0487,0,10000181
-0,2013-09-20,FP-GTC,95173,0,-99,42.2822,0,10000642
-0,2013-01-02,Auction,43479,0,5,21.7236,0,10000900
-0,2012-11-18,FP-GTC,139973,0,11,9.9099,0,10000191
-0,2012-02-03,Others,40059,3,5,74.0231,0,10000422
-0,2012-04-28,FP-non GTC,3838,0,5,6.5803,0,10000162
-0,2013-05-23,Others,80287,0,14,79.3278,0,10000474
-0,2012-06-16,Auction,108782,15,13,15.8824,0,10000259
-0,2013-12-05,FP-non GTC,63861,0,-99,78.2662,0,10000510
-0,2013-02-14,ABIN,31519,0,5,98.5361,0,10000591
-0,2012-07-27,ABIN,38238,0,16,97.9582,0,10000396
-0,2012-09-20,FP-non GTC,31519,0,13,15.0370,0,10000080
-0,2013-09-16,Auction,45333,0,15,99.9888,0,10000082
-0,2013-03-23,ABIN,1357,0,-99,78.1418,0,10000091
-0,2013-04-11,FP-GTC,45333,0,11,58.2670,0,10000078
-0,2012-01-23,Auction,63864,3,12,78.3096,0,10000755
-0,2012-01-27,FP-non GTC,15115,0,13,62.3493,0,10000191
-0,2012-10-30,FP-non GTC,1357,0,16,75.3922,0,10000178
-0,2012-05-22,Auction,75665,0,16,25.2559,0,10000662
-0,2013-08-22,Auction,51582,0,11,22.2512,0,10000937
-0,2012-10-20,ABIN,139973,0,-99,5.9459,0,10000542
-0,2013-04-06,FP-GTC,15868,0,5,44.5247,0,10000090
-0,2012-09-06,Others,43479,0,13,22.0043,0,10000822
-0,2012-05-19,ABIN,31519,0,12,67.8089,0,10000046
-0,2012-08-13,Auction,57990,3,12,51.2947,0,10000472
-0,2012-09-18,FP-GTC,139973,0,15,50.6074,0,10000482
-0,2013-02-27,Auction,13836,0,5,12.9600,0,10000057
-0,2012-10-02,Others,53064,0,11,21.0200,0,10000142
-0,2013-02-23,FP-GTC,26262,0,11,18.1814,0,10000457
-0,2012-10-05,FP-non GTC,100847,0,16,17.3068,0,10000061
-0,2013-05-03,FP-GTC,99985,0,11,88.9624,0,10000299
-0,2012-05-22,Others,11848,0,13,25.6477,0,10000572
-0,2013-01-21,ABIN,33038,15,16,37.8162,0,10000036
-0,2013-09-22,FP-non GTC,57990,0,12,72.4322,0,10000304
-0,2013-09-22,ABIN,65,0,13,5.8659,0,10000424
-0,2012-08-09,ABIN,44079,0,5,65.9028,0,10000790
-0,2013-08-05,Auction,1504,0,13,87.7795,0,10000621
-0,2013-08-06,FP-non GTC,88750,0,5,47.7266,0,10000579
-0,2013-11-22,Auction,62179,0,16,17.7918,0,10000626
-0,2013-08-19,ABIN,16145,0,16,56.5899,0,10000818
-0,2012-09-24,ABIN,4943,0,16,85.8548,0,10000515
-0,2013-05-14,ABIN,43479,0,5,12.9686,0,10000527
-0,2013-07-15,Others,73506,0,-99,88.5967,0,10000266
-0,2012-11-25,FP-GTC,64076,0,14,28.2347,0,10000361
-0,2012-01-07,ABIN,80053,0,13,77.0661,0,10000149
-0,2013-06-08,FP-non GTC,80287,0,-99,98.5548,0,10000668
-0,2013-10-03,ABIN,88750,0,11,34.1582,0,10000116
-0,2012-04-26,Auction,175750,0,12,53.9184,0,10000084
-0,2013-01-31,FP-non GTC,15868,0,12,71.5902,0,10000372
-0,2013-04-23,FP-non GTC,32876,0,14,94.0781,0,10000798
-0,2013-10-20,ABIN,38238,0,13,43.3655,0,10000837
-0,2013-04-21,ABIN,1504,0,5,32.5360,0,10000954
-0,2013-03-05,FP-non GTC,156356,0,11,48.8941,0,10000480
-0,2013-05-08,FP-non GTC,139973,0,16,94.5155,0,10000278
-0,2013-09-06,FP-GTC,45333,0,14,93.0745,0,10000821
-0,201
<TRUNCATED>
[23/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/EDW.TEST_CAL_DT.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_CAL_DT.csv b/examples/test_case_data/localmeta/data/EDW.TEST_CAL_DT.csv
deleted file mode 100644
index 67efe23..0000000
--- a/examples/test_case_data/localmeta/data/EDW.TEST_CAL_DT.csv
+++ /dev/null
@@ -1,731 +0,0 @@
-2012-08-16,2012-01-01,2012-07-01,2012-08-01,2012-08-11,0,-1,-3,-15,-103,0,-1,-4,-15,-15,41501,228,47,16,6,33,5928,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-17,16-Aug-2012,Aug 16th 2012,Fri 08-16-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-11,2012-08-12,2012-08-16,Fri ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,33,2012,2012-08-11,2012-08-17,N,Wk.33 - 13,2012-08-11 00:00:00,2012-08-17 00:00:00,2012W33 ,2012W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2012,N,2012-08-16,2011-08-16,2012-05-16,2012-02-16,2012-07-16,2012-06-16,2012-08-09,2012-08-02,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-03,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-328,0,-3,-11,-47,-47,41276,3,3,3,5,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,03-Jan-2012,Jan 3rd 2012,Thu 01-03-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-03,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-03,2011-01-03,2012-10-03,2012-07-03,2012-12-03,2012-11-03,2012-12-27,2012-12-20,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-10,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-231,0,-2,-8,-33,-33,41373,100,10,10,4,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,10-Apr-2012,Apr 10th 2012,Wed 04-10-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-10,Wed ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-10,2011-04-10,2012-01-10,2012-10-10,2012-03-10,2012-02-10,2012-04-03,2012-03-27,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-12,2012-01-01,2012-04-01,2012-06-01,2012-06-09,0,-2,-5,-24,-168,0,-2,-6,-24,-24,41436,163,73,12,4,24,5919,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-15,12-Jun-2012,Jun 12th 2012,Wed 06-12-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-09,2012-06-10,2012-06-12,Wed ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,24,2012,2012-06-09,2012-06-15,N,Wk.24 - 13,2012-06-09 00:00:00,2012-06-15 00:00:00,2012W24 ,2012W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2012,N,2012-06-12,2011-06-12,2012-03-12,2012-12-12,2012-05-12,2012-04-12,2012-06-05,2012-05-29,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-27,2012-01-01,2012-04-01,2012-04-01,2012-04-21,0,-2,-7,-31,-214,0,-2,-8,-31,-31,41390,117,27,27,7,17,5912,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-27,27-Apr-2012,Apr 27th 2012,Sat 04-27-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-21,2012-04-22,2012-04-27,Sat ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,17,2012,2012-04-21,2012-04-27,N,Wk.17 - 13,2012-04-21 00:00:00,2012-04-27 00:00:00,2012W17 ,2012W17 ,04/21/13 - 04/27/13,04/21 - 04/27,2012,N,2012-04-27,2011-04-27,2012-01-27,2012-10-27,2012-03-27,2012-02-27,2012-04-20,2012-04-13,0,0,0,0,0,0,0,0,4,2,17,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-19,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-69,0,-1,-3,-10,-10,41535,262,81,19,5,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,19-Sep-2012,Sep 19th 2012,Thu 09-19-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-19,Thu ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-19,2011-09-19,2012-06-19,2012-03-19,2012-08-19,2012-07-19,2012-09-12,2012-09-05,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-07,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-265,0,-3,-9,-38,-38,41339,66,66,7,5,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,07-Mar-2012,Mar 7th 2012,Thu 03-07-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-07,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-07,2011-03-07,2012-12-07,2012-09-07,2012-02-07,2012-01-07,2012-02-28,2012-02-21,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-28,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-244,0,-3,-9,-35,-35,41360,87,87,28,5,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,28-Mar-2012,Mar 28th 2012,Thu 03-28-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-28,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-28,2011-03-28,2012-12-28,2012-09-28,2012-02-28,2012-01-28,2012-03-21,2012-03-14,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-23,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-4,0,0,-1,-1,-1,41600,327,54,23,7,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,23-Nov-2012,Nov 23rd 2012,Sat 11-23-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-23,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-23,2011-11-23,2012-08-23,2012-05-23,2012-10-23,2012-09-23,2012-11-16,2012-11-09,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-09,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-263,0,-3,-9,-38,-38,41341,68,68,9,7,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,09-Mar-2012,Mar 9th 2012,Sat 03-09-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-09,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-09,2011-03-09,2012-12-09,2012-09-09,2012-02-09,2012-01-09,2012-03-02,2012-02-23,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-21,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-6,0,0,-1,-1,-1,41598,325,52,21,5,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,21-Nov-2012,Nov 21st 2012,Thu 11-21-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-21,Thu ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-21,2011-11-21,2012-08-21,2012-05-21,2012-10-21,2012-09-21,2012-11-14,2012-11-07,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-26,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-246,0,-3,-9,-35,-35,41358,85,85,26,3,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,26-Mar-2012,Mar 26th 2012,Tue 03-26-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-26,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-26,2011-03-26,2012-12-26,2012-09-26,2012-02-26,2012-01-26,2012-03-19,2012-03-12,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-24,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-307,0,-3,-11,-44,-44,41297,24,24,24,5,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,24-Jan-2012,Jan 24th 2012,Thu 01-24-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-24,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-24,2011-01-24,2012-10-24,2012-07-24,2012-12-24,2012-11-24,2012-01-17,2012-01-10,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-01,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-330,0,-3,-11,-47,-47,41274,1,1,1,3,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,01-Jan-2012,Jan 1st 2012,Tue 01-01-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-01,Tue ,2012M01,Jan-2012,Y,2012M01 ,Y,Year 2012 - Quarter 01,2012Q01 ,Y,1,2012,2012-12-30,2012-01-05,Y,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,Y,2012-01-01,2011-01-01,2012-10-01,2012-07-01,2012-12-01,2012-11-01,2012-12-25,2012-12-18,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-29,2012-01-01,2012-04-01,2012-04-01,2012-04-28,0,-2,-7,-30,-212,0,-2,-7,-30,-30,41392,119,29,29,2,18,5913,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-05-04,29-Apr-2012,Apr 29th 2012,Mon 04-29-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-04-28,2012-04-28,2012-04-29,2012-04-29,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,18,2012,2012-04-28,2012-05-04,N,Wk.18 - 13,2012-04-28 00:00:00,2012-05-04 00:00:00,2012W18 ,2012W18 ,04/28/13 - 05/04/13,04/28 - 05/04,2012,N,2012-04-29,2011-04-29,2012-01-29,2012-10-29,2012-03-29,2012-02-28,2012-04-22,2012-04-15,0,0,0,0,0,0,0,0,5,2,18,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-11,2012-01-01,2012-04-01,2012-05-01,2012-05-05,0,-2,-6,-29,-200,0,-2,-7,-29,-29,41404,131,41,11,7,19,5914,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-11,11-May-2012,May 11th 2012,Sat 05-11-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-05,2012-05-06,2012-05-11,Sat ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,19,2012,2012-05-05,2012-05-11,N,Wk.19 - 13,2012-05-05 00:00:00,2012-05-11 00:00:00,2012W19 ,2012W19 ,05/05/13 - 05/11/13,05/05 - 05/11,2012,N,2012-05-11,2011-05-11,2012-02-11,2012-11-11,2012-04-11,2012-03-11,2012-05-04,2012-04-27,0,0,0,0,0,0,0,0,5,2,19,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-01,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-57,0,0,-2,-8,-8,41547,274,1,1,3,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,01-Oct-2012,Oct 1st 2012,Tue 10-01-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-01,Tue ,2012M10,Oct-2012,Y,2012M10 ,Y,Year 2012 - Quarter 04,2012Q04 ,Y,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-01,2011-10-01,2012-07-01,2012-04-01,2012-09-01,2012-08-01,2012-09-24,2012-09-17,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-05,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,8,0,0,0,1,1,41612,339,66,5,5,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,05-Dec-2012,Dec 5th 2012,Thu 12-05-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-05,Thu ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-05,2011-12-05,2012-09-05,2012-06-05,2012-11-05,2012-10-05,2012-11-28,2012-11-21,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-05,2012-01-01,2012-01-01,2012-03-01,2012-03-03,0,-3,-8,-38,-267,0,-3,-9,-38,-38,41337,64,64,5,3,10,5905,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-09,05-Mar-2012,Mar 5th 2012,Tue 03-05-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-03,2012-03-04,2012-03-05,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,10,2012,2012-03-03,2012-03-09,N,Wk.10 - 13,2012-03-03 00:00:00,2012-03-09 00:00:00,2012W10 ,2012W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2012,N,2012-03-05,2011-03-05,2012-12-05,2012-09-05,2012-02-05,2012-01-05,2012-02-26,2012-02-19,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-03,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-55,0,0,-2,-8,-8,41549,276,3,3,5,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,03-Oct-2012,Oct 3rd 2012,Thu 10-03-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-03,Thu ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-03,2011-10-03,2012-07-03,2012-04-03,2012-09-03,2012-08-03,2012-09-26,2012-09-19,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-22,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-309,0,-3,-11,-44,-44,41295,22,22,22,3,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,22-Jan-2012,Jan 22nd 2012,Tue 01-22-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-22,Tue ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-22,2011-01-22,2012-10-22,2012-07-22,2012-12-22,2012-11-22,2012-01-15,2012-01-08,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-26,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,29,0,0,0,4,4,41633,360,87,26,5,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,26-Dec-2012,Dec 26th 2012,Thu 12-26-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-26,Thu ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-26,2011-12-26,2012-09-26,2012-06-26,2012-11-26,2012-10-26,2012-12-19,2012-12-12,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-17,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-71,0,-1,-3,-10,-10,41533,260,79,17,3,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,17-Sep-2012,Sep 17th 2012,Tue 09-17-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-17,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-17,2011-09-17,2012-06-17,2012-03-17,2012-08-17,2012-07-17,2012-09-10,2012-09-03,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-20,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-38,0,0,-2,-5,-6,41566,293,20,20,1,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,20-Oct-2012,Oct 20th 2012,Sun 10-20-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-14,2012-10-20,Sun ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,Y,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-20,2011-10-20,2012-07-20,2012-04-20,2012-09-20,2012-08-20,2012-10-13,2012-10-06,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-25,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,-2,0,0,0,0,0,41602,329,56,25,2,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,25-Nov-2012,Nov 25th 2012,Mon 11-25-13,1,1,1,1,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-25,2012-11-25,Mon ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,N,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-25,2011-11-25,2012-08-25,2012-05-25,2012-10-25,2012-09-25,2012-11-18,2012-11-11,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-26,2012-01-01,2012-01-01,2012-01-01,2012-01-20,0,-3,-10,-44,-305,0,-3,-11,-44,-44,41299,26,26,26,7,4,5899,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-26,26-Jan-2012,Jan 26th 2012,Sat 01-26-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-20,2012-01-21,2012-01-26,Sat ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,4,2012,2012-01-20,2012-01-26,N,Wk.04 - 13,2012-01-20 00:00:00,2012-01-26 00:00:00,2012W04 ,2012W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2012,N,2012-01-26,2011-01-26,2012-10-26,2012-07-26,2012-12-26,2012-11-26,2012-01-19,2012-01-12,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-24,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,27,0,0,0,4,4,41631,358,85,24,3,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,24-Dec-2012,Dec 24th 2012,Tue 12-24-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-24,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-24,2011-12-24,2012-09-24,2012-06-24,2012-11-24,2012-10-24,2012-12-17,2012-12-10,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-04,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-296,0,-3,-10,-42,-42,41308,35,35,4,2,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,04-Feb-2012,Feb 4th 2012,Mon 02-04-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-04,Mon ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-04,2011-02-04,2012-11-04,2012-08-04,2012-01-04,2012-12-04,2012-01-28,2012-01-21,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-30,2012-01-01,2012-04-01,2012-05-01,2012-05-26,0,-2,-6,-26,-181,0,-2,-6,-26,-26,41423,150,60,30,5,22,5917,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-06-01,30-May-2012,May 30th 2012,Thu 05-30-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-05-26,2012-05-26,2012-05-27,2012-05-30,Thu ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,22,2012,2012-05-26,2012-06-01,N,Wk.22 - 13,2012-05-26 00:00:00,2012-06-01 00:00:00,2012W22 ,2012W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2012,N,2012-05-30,2011-05-30,2012-02-28,2012-11-30,2012-04-30,2012-03-30,2012-05-23,2012-05-16,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-12,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-229,0,-2,-8,-33,-33,41375,102,12,12,6,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,12-Apr-2012,Apr 12th 2012,Fri 04-12-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-12,Fri ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-12,2011-04-12,2012-01-12,2012-10-12,2012-03-12,2012-02-12,2012-04-05,2012-03-29,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-08,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-233,0,-2,-8,-33,-33,41371,98,8,8,2,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,08-Apr-2012,Apr 8th 2012,Mon 04-08-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-08,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-08,2011-04-08,2012-01-08,2012-10-08,2012-03-08,2012-02-08,2012-04-01,2012-03-25,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-28,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,31,0,0,0,4,4,41635,362,89,28,7,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,28-Dec-2012,Dec 28th 2012,Sat 12-28-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-28,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-28,2011-12-28,2012-09-28,2012-06-28,2012-11-28,2012-10-28,2012-12-21,2012-12-14,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-03,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,6,0,0,0,1,1,41610,337,64,3,3,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,03-Dec-2012,Dec 3rd 2012,Tue 12-03-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-03,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-03,2011-12-03,2012-09-03,2012-06-03,2012-11-03,2012-10-03,2012-11-26,2012-11-19,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-15,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-135,0,-1,-5,-19,-19,41469,196,15,15,2,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,15-Jul-2012,Jul 15th 2012,Mon 07-15-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-15,Mon ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-15,2011-07-15,2012-04-15,2012-01-15,2012-06-15,2012-05-15,2012-07-08,2012-07-01,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-13,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-198,0,-2,-7,-28,-28,41406,133,43,13,2,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,13-May-2012,May 13th 2012,Mon 05-13-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-13,Mon ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-13,2011-05-13,2012-02-13,2012-11-13,2012-04-13,2012-03-13,2012-05-06,2012-04-29,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-30,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-242,0,-3,-9,-35,-35,41362,89,89,30,7,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,30-Mar-2012,Mar 30th 2012,Sat 03-30-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-30,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-30,2011-03-30,2012-12-30,2012-09-30,2012-02-28,2012-01-30,2012-03-23,2012-03-16,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-09,2012-01-01,2012-10-01,2012-12-01,2012-12-08,0,0,1,2,12,0,0,0,2,2,41616,343,70,9,2,50,5945,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-14,09-Dec-2012,Dec 9th 2012,Mon 12-09-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-08,2012-12-09,2012-12-09,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,50,2012,2012-12-08,2012-12-14,N,Wk.50 - 13,2012-12-08 00:00:00,2012-12-14 00:00:00,2012W50 ,2012W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2012,N,2012-12-09,2011-12-09,2012-09-09,2012-06-09,2012-11-09,2012-10-09,2012-12-02,2012-11-25,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-25,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-275,0,-3,-9,-39,-39,41329,56,56,25,2,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,25-Feb-2012,Feb 25th 2012,Mon 02-25-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-02-25,Mon ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-02-25,2011-02-25,2012-11-25,2012-08-25,2012-01-25,2012-12-25,2012-02-18,2012-02-11,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-04,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-23,0,0,-1,-3,-3,41581,308,35,4,2,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,04-Nov-2012,Nov 4th 2012,Mon 11-04-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-04,Mon ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-04,2011-11-04,2012-08-04,2012-05-04,2012-10-04,2012-09-04,2012-10-28,2012-10-21,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-08,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-19,0,0,-1,-3,-3,41585,312,39,8,6,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,08-Nov-2012,Nov 8th 2012,Fri 11-08-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-08,Fri ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-08,2011-11-08,2012-08-08,2012-05-08,2012-10-08,2012-09-08,2012-11-01,2012-10-25,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-22,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-97,0,-1,-4,-14,-14,41507,234,53,22,5,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,22-Aug-2012,Aug 22nd 2012,Thu 08-22-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-22,Thu ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-22,2011-08-22,2012-05-22,2012-02-22,2012-07-22,2012-06-22,2012-08-15,2012-08-08,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-18,2012-01-01,2012-04-01,2012-06-01,2012-06-16,0,-2,-5,-23,-162,0,-2,-6,-23,-23,41442,169,79,18,3,25,5920,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-22,18-Jun-2012,Jun 18th 2012,Tue 06-18-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-16,2012-06-17,2012-06-18,Tue ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,25,2012,2012-06-16,2012-06-22,N,Wk.25 - 13,2012-06-16 00:00:00,2012-06-22 00:00:00,2012W25 ,2012W25 ,06/16/13 - 06/22/13,06/16 - 06/22,2012,N,2012-06-18,2011-06-18,2012-03-18,2012-12-18,2012-05-18,2012-04-18,2012-06-11,2012-06-04,0,0,0,0,0,0,0,0,6,2,25,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-19,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-131,0,-1,-5,-19,-19,41473,200,19,19,6,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,19-Jul-2012,Jul 19th 2012,Fri 07-19-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-19,Fri ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-19,2011-07-19,2012-04-19,2012-01-19,2012-06-19,2012-05-19,2012-07-12,2012-07-05,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-20,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-99,0,-1,-4,-14,-14,41505,232,51,20,3,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,20-Aug-2012,Aug 20th 2012,Tue 08-20-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-20,Tue ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-20,2011-08-20,2012-05-20,2012-02-20,2012-07-20,2012-06-20,2012-08-13,2012-08-06,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-24,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-95,0,-1,-4,-14,-14,41509,236,55,24,7,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,24-Aug-2012,Aug 24th 2012,Sat 08-24-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-24,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-24,2011-08-24,2012-05-24,2012-02-24,2012-07-24,2012-06-24,2012-08-17,2012-08-10,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-13,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-137,0,-1,-5,-20,-20,41467,194,13,13,7,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,13-Jul-2012,Jul 13th 2012,Sat 07-13-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-13,Sat ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-13,2011-07-13,2012-04-13,2012-01-13,2012-06-13,2012-05-13,2012-07-06,2012-06-29,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-06,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-21,0,0,-1,-3,-3,41583,310,37,6,4,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,06-Nov-2012,Nov 6th 2012,Wed 11-06-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-06,Wed ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-06,2011-11-06,2012-08-06,2012-05-06,2012-10-06,2012-09-06,2012-10-30,2012-10-23,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-07,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,10,0,0,0,1,1,41614,341,68,7,7,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,07-Dec-2012,Dec 7th 2012,Sat 12-07-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-07,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-07,2011-12-07,2012-09-07,2012-06-07,2012-11-07,2012-10-07,2012-11-30,2012-11-23,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-04,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-146,0,-1,-5,-21,-21,41458,185,4,4,5,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,04-Jul-2012,Jul 4th 2012,Thu 07-04-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-04,Thu ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-04,2011-07-04,2012-04-04,2012-01-04,2012-06-04,2012-05-04,2012-06-27,2012-06-20,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-07,2012-01-01,2012-10-01,2012-10-01,2012-10-06,0,0,-1,-7,-51,0,0,-2,-7,-7,41553,280,7,7,2,41,5936,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-12,07-Oct-2012,Oct 7th 2012,Mon 10-07-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-06,2012-10-07,2012-10-07,Mon ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,41,2012,2012-10-06,2012-10-12,N,Wk.41 - 13,2012-10-06 00:00:00,2012-10-12 00:00:00,2012W41 ,2012W41 ,10/06/13 - 10/12/13,10/06 - 10/12,2012,N,2012-10-07,2011-10-07,2012-07-07,2012-04-07,2012-09-07,2012-08-07,2012-09-30,2012-09-23,0,0,0,0,0,0,0,0,10,4,41,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-10,2012-01-01,2012-10-01,2012-11-01,2012-11-10,0,0,0,-2,-17,0,0,-1,-2,-3,41587,314,41,10,1,46,5941,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-16,10-Nov-2012,Nov 10th 2012,Sun 11-10-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-10,2012-11-04,2012-11-10,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,46,2012,2012-11-10,2012-11-16,Y,Wk.46 - 13,2012-11-10 00:00:00,2012-11-16 00:00:00,2012W46 ,2012W46 ,11/10/13 - 11/16/13,11/10 - 11/16,2012,N,2012-11-10,2011-11-10,2012-08-10,2012-05-10,2012-10-10,2012-09-10,2012-11-03,2012-10-27,0,0,0,0,0,0,0,0,11,4,46,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-07,2012-01-01,2012-07-01,2012-08-01,2012-08-04,0,-1,-3,-16,-112,0,-1,-4,-16,-16,41492,219,38,7,4,32,5927,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-10,07-Aug-2012,Aug 7th 2012,Wed 08-07-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-04,2012-08-05,2012-08-07,Wed ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,32,2012,2012-08-04,2012-08-10,N,Wk.32 - 13,2012-08-04 00:00:00,2012-08-10 00:00:00,2012W32 ,2012W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2012,N,2012-08-07,2011-08-07,2012-05-07,2012-02-07,2012-07-07,2012-06-07,2012-07-31,2012-07-24,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-25,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-63,0,-1,-3,-9,-9,41541,268,87,25,4,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,25-Sep-2012,Sep 25th 2012,Wed 09-25-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-23,2012-09-25,Wed ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,N,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-25,2011-09-25,2012-06-25,2012-03-25,2012-08-25,2012-07-25,2012-09-18,2012-09-11,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-05,2012-01-01,2012-10-01,2012-10-01,2012-09-29,0,0,-1,-8,-53,0,0,-2,-8,-8,41551,278,5,5,7,40,5935,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-05,05-Oct-2012,Oct 5th 2012,Sat 10-05-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-09-29,2012-09-30,2012-10-05,Sat ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,40,2012,2012-09-29,2012-10-05,N,Wk.40 - 13,2012-09-29 00:00:00,2012-10-05 00:00:00,2012W40 ,2012W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2012,N,2012-10-05,2011-10-05,2012-07-05,2012-04-05,2012-09-05,2012-08-05,2012-09-28,2012-09-21,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-02,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-86,0,-1,-3,-12,-12,41518,245,64,2,2,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,02-Sep-2012,Sep 2nd 2012,Mon 09-02-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-09-02,2012-09-02,Mon ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,N,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-02,2011-09-02,2012-06-02,2012-03-02,2012-08-02,2012-07-02,2012-08-26,2012-08-19,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-15,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-196,0,-2,-7,-28,-28,41408,135,45,15,4,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,15-May-2012,May 15th 2012,Wed 05-15-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-15,Wed ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-15,2011-05-15,2012-02-15,2012-11-15,2012-04-15,2012-03-15,2012-05-08,2012-05-01,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-10,2012-01-01,2012-01-01,2012-02-01,2012-02-10,0,-3,-9,-41,-290,0,-3,-10,-41,-42,41314,41,41,10,1,7,5902,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-16,10-Feb-2012,Feb 10th 2012,Sun 02-10-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-10,2012-02-04,2012-02-10,Sun ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,7,2012,2012-02-10,2012-02-16,Y,Wk.07 - 13,2012-02-10 00:00:00,2012-02-16 00:00:00,2012W07 ,2012W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2012,N,2012-02-10,2011-02-10,2012-11-10,2012-08-10,2012-01-10,2012-12-10,2012-02-03,2012-01-27,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-21,2012-01-01,2012-10-01,2012-12-01,2012-12-15,0,0,1,3,24,0,0,0,3,3,41628,355,82,21,7,51,5946,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-21,21-Dec-2012,Dec 21st 2012,Sat 12-21-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-15,2012-12-16,2012-12-21,Sat ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,51,2012,2012-12-15,2012-12-21,N,Wk.51 - 13,2012-12-15 00:00:00,2012-12-21 00:00:00,2012W51 ,2012W51 ,12/15/13 - 12/21/13,12/15 - 12/21,2012,N,2012-12-21,2011-12-21,2012-09-21,2012-06-21,2012-11-21,2012-10-21,2012-12-14,2012-12-07,0,0,0,0,0,0,0,0,12,4,51,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-22,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-36,0,0,-2,-5,-5,41568,295,22,22,3,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,22-Oct-2012,Oct 22nd 2012,Tue 10-22-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-22,Tue ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-22,2011-10-22,2012-07-22,2012-04-22,2012-09-22,2012-08-22,2012-10-15,2012-10-08,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-24,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-34,0,0,-2,-5,-5,41570,297,24,24,5,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,24-Oct-2012,Oct 24th 2012,Thu 10-24-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-24,Thu ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-24,2011-10-24,2012-07-24,2012-04-24,2012-09-24,2012-08-24,2012-10-17,2012-10-10,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-10,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-140,0,-1,-5,-20,-20,41464,191,10,10,4,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,10-Jul-2012,Jul 10th 2012,Wed 07-10-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-10,Wed ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-10,2011-07-10,2012-04-10,2012-01-10,2012-06-10,2012-05-10,2012-07-03,2012-06-26,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-27,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-273,0,-3,-9,-39,-39,41331,58,58,27,4,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,27-Feb-2012,Feb 27th 2012,Wed 02-27-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-02-27,Wed ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-02-27,2011-02-27,2012-11-27,2012-08-27,2012-01-27,2012-12-27,2012-02-20,2012-02-13,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-02,2012-01-01,2012-01-01,2012-03-01,2012-02-24,0,-3,-8,-39,-270,0,-3,-9,-39,-39,41334,61,61,2,7,9,5904,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-02,02-Mar-2012,Mar 2nd 2012,Sat 03-02-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-25,2012-03-02,Sat ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,N,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2012-03-02,2011-03-02,2012-12-02,2012-09-02,2012-02-02,2012-01-02,2012-02-23,2012-02-16,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-02,2012-01-01,2012-10-01,2012-11-01,2012-10-27,0,0,0,-4,-25,0,0,-1,-4,-4,41579,306,33,2,7,44,5939,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-02,02-Nov-2012,Nov 2nd 2012,Sat 11-02-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-10-27,2012-10-28,2012-11-02,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,44,2012,2012-10-27,2012-11-02,N,Wk.44 - 13,2012-10-27 00:00:00,2012-11-02 00:00:00,2012W44 ,2012W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2012,N,2012-11-02,2011-11-02,2012-08-02,2012-05-02,2012-10-02,2012-09-02,2012-10-26,2012-10-19,0,0,0,0,0,0,0,0,11,4,44,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-21,2012-01-01,2012-01-01,2012-03-01,2012-03-17,0,-3,-8,-36,-251,0,-3,-9,-36,-36,41353,80,80,21,5,12,5907,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-23,21-Mar-2012,Mar 21st 2012,Thu 03-21-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-17,2012-03-18,2012-03-21,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,12,2012,2012-03-17,2012-03-23,N,Wk.12 - 13,2012-03-17 00:00:00,2012-03-23 00:00:00,2012W12 ,2012W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2012,N,2012-03-21,2011-03-21,2012-12-21,2012-09-21,2012-02-21,2012-01-21,2012-03-14,2012-03-07,0,0,0,0,0,0,0,0,3,1,12,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-03,2012-01-01,2012-04-01,2012-04-01,2012-03-31,0,-2,-7,-34,-238,0,-2,-8,-34,-34,41366,93,3,3,4,14,5909,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-06,03-Apr-2012,Apr 3rd 2012,Wed 04-03-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-03-31,2012-04-01,2012-04-03,Wed ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,14,2012,2012-03-31,2012-04-06,N,Wk.14 - 13,2012-03-31 00:00:00,2012-04-06 00:00:00,2012W14 ,2012W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2012,N,2012-04-03,2011-04-03,2012-01-03,2012-10-03,2012-03-03,2012-02-03,2012-03-27,2012-03-20,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-06,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-144,0,-1,-5,-21,-21,41460,187,6,6,7,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,06-Jul-2012,Jul 6th 2012,Sat 07-06-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-06,Sat ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-06,2011-07-06,2012-04-06,2012-01-06,2012-06-06,2012-05-06,2012-06-29,2012-06-22,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-26,2012-01-01,2012-04-01,2012-06-01,2012-06-23,0,-2,-5,-22,-154,0,-2,-6,-22,-22,41450,177,87,26,4,26,5921,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-29,26-Jun-2012,Jun 26th 2012,Wed 06-26-13,1,0,0,0,2012-06-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-23,2012-06-24,2012-06-26,Wed ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,26,2012,2012-06-23,2012-06-29,N,Wk.26 - 13,2012-06-23 00:00:00,2012-06-29 00:00:00,2012W26 ,2012W26 ,06/23/13 - 06/29/13,06/23 - 06/29,2012,N,2012-06-26,2011-06-26,2012-03-26,2012-12-26,2012-05-26,2012-04-26,2012-06-19,2012-06-12,0,0,0,0,0,0,0,0,6,2,26,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-26,2012-01-01,2012-07-01,2012-08-01,2012-08-25,0,-1,-3,-13,-93,0,-1,-3,-13,-13,41511,238,57,26,2,35,5930,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-31,26-Aug-2012,Aug 26th 2012,Mon 08-26-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-08-25,2012-08-25,2012-08-26,2012-08-26,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,35,2012,2012-08-25,2012-08-31,N,Wk.35 - 13,2012-08-25 00:00:00,2012-08-31 00:00:00,2012W35 ,2012W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2012,N,2012-08-26,2011-08-26,2012-05-26,2012-02-26,2012-07-26,2012-06-26,2012-08-19,2012-08-12,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-13,2012-01-01,2012-10-01,2012-12-01,2012-12-08,0,0,1,2,16,0,0,0,2,2,41620,347,74,13,6,50,5945,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-14,13-Dec-2012,Dec 13th 2012,Fri 12-13-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-08,2012-12-09,2012-12-13,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,50,2012,2012-12-08,2012-12-14,N,Wk.50 - 13,2012-12-08 00:00:00,2012-12-14 00:00:00,2012W50 ,2012W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2012,N,2012-12-13,2011-12-13,2012-09-13,2012-06-13,2012-11-13,2012-10-13,2012-12-06,2012-11-29,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-02,2012-01-01,2012-07-01,2012-07-01,2012-06-30,0,-1,-4,-21,-148,0,-1,-5,-21,-21,41456,183,2,2,3,27,5922,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-06,02-Jul-2012,Jul 2nd 2012,Tue 07-02-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-06-30,2012-07-01,2012-07-02,Tue ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,27,2012,2012-06-30,2012-07-06,N,Wk.27 - 13,2012-06-30 00:00:00,2012-07-06 00:00:00,2012W27 ,2012W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2012,N,2012-07-02,2011-07-02,2012-04-02,2012-01-02,2012-06-02,2012-05-02,2012-06-25,2012-06-18,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-05,2012-01-01,2012-07-01,2012-08-01,2012-08-04,0,-1,-3,-16,-114,0,-1,-4,-16,-16,41490,217,36,5,2,32,5927,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-10,05-Aug-2012,Aug 5th 2012,Mon 08-05-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-04,2012-08-05,2012-08-05,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,32,2012,2012-08-04,2012-08-10,N,Wk.32 - 13,2012-08-04 00:00:00,2012-08-10 00:00:00,2012W32 ,2012W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2012,N,2012-08-05,2011-08-05,2012-05-05,2012-02-05,2012-07-05,2012-06-05,2012-07-29,2012-07-22,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-08,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-292,0,-3,-10,-42,-42,41312,39,39,8,6,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,08-Feb-2012,Feb 8th 2012,Fri 02-08-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-08,Fri ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-08,2011-02-08,2012-11-08,2012-08-08,2012-01-08,2012-12-08,2012-02-01,2012-01-25,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-14,2012-01-01,2012-01-01,2012-02-01,2012-02-10,0,-3,-9,-41,-286,0,-3,-10,-41,-41,41318,45,45,14,5,7,5902,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-16,14-Feb-2012,Feb 14th 2012,Thu 02-14-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-10,2012-02-11,2012-02-14,Thu ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,7,2012,2012-02-10,2012-02-16,N,Wk.07 - 13,2012-02-10 00:00:00,2012-02-16 00:00:00,2012W07 ,2012W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2012,N,2012-02-14,2011-02-14,2012-11-14,2012-08-14,2012-01-14,2012-12-14,2012-02-07,2012-01-31,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-21,2012-01-01,2012-07-01,2012-09-01,2012-09-15,0,-1,-2,-10,-67,0,-1,-3,-10,-10,41537,264,83,21,7,38,5933,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-21,21-Sep-2012,Sep 21st 2012,Sat 09-21-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-15,2012-09-16,2012-09-21,Sat ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,38,2012,2012-09-15,2012-09-21,N,Wk.38 - 13,2012-09-15 00:00:00,2012-09-21 00:00:00,2012W38 ,2012W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2012,N,2012-09-21,2011-09-21,2012-06-21,2012-03-21,2012-08-21,2012-07-21,2012-09-14,2012-09-07,0,0,0,0,0,0,0,0,9,3,38,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-19,2012-01-01,2012-01-01,2012-03-01,2012-03-17,0,-3,-8,-36,-253,0,-3,-9,-36,-36,41351,78,78,19,3,12,5907,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-23,19-Mar-2012,Mar 19th 2012,Tue 03-19-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-17,2012-03-18,2012-03-19,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,12,2012,2012-03-17,2012-03-23,N,Wk.12 - 13,2012-03-17 00:00:00,2012-03-23 00:00:00,2012W12 ,2012W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2012,N,2012-03-19,2011-03-19,2012-12-19,2012-09-19,2012-02-19,2012-01-19,2012-03-12,2012-03-05,0,0,0,0,0,0,0,0,3,1,12,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-28,2012-01-01,2012-01-01,2012-01-01,2012-01-27,0,-3,-10,-43,-303,0,-3,-10,-43,-43,41301,28,28,28,2,5,5900,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-02-02,28-Jan-2012,Jan 28th 2012,Mon 01-28-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-01-27,2012-01-27,2012-01-28,2012-01-28,Mon ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,5,2012,2012-01-27,2012-02-02,N,Wk.05 - 13,2012-01-27 00:00:00,2012-02-02 00:00:00,2012W05 ,2012W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2012,N,2012-01-28,2011-01-28,2012-10-28,2012-07-28,2012-12-28,2012-11-28,2012-01-21,2012-01-14,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-03,2012-01-01,2012-07-01,2012-08-01,2012-07-28,0,-1,-3,-17,-116,0,-1,-4,-17,-17,41488,215,34,3,7,31,5926,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-03,03-Aug-2012,Aug 3rd 2012,Sat 08-03-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-07-28,2012-07-29,2012-08-03,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,31,2012,2012-07-28,2012-08-03,N,Wk.31 - 13,2012-07-28 00:00:00,2012-08-03 00:00:00,2012W31 ,2012W31 ,07/28/13 - 08/03/13,07/28 - 08/03,2012,N,2012-08-03,2011-08-03,2012-05-03,2012-02-03,2012-07-03,2012-06-03,2012-07-27,2012-07-20,0,0,0,0,0,0,0,0,8,3,31,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-06-14,2012-01-01,2012-04-01,2012-06-01,2012-06-09,0,-2,-5,-24,-166,0,-2,-6,-24,-24,41438,165,75,14,6,24,5919,3,6,1362,2,454,114,2012-12-31,2012-06-30,2012-06-30,2012-06-15,14-Jun-2012,Jun 14th 2012,Fri 06-14-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-05-26,2012-06-09,2012-06-10,2012-06-14,Fri ,2012M06,Jun-2012,N,2012M06 ,N,Year 2012 - Quarter 02,2012Q02 ,N,24,2012,2012-06-09,2012-06-15,N,Wk.24 - 13,2012-06-09 00:00:00,2012-06-15 00:00:00,2012W24 ,2012W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2012,N,2012-06-14,2011-06-14,2012-03-14,2012-12-14,2012-05-14,2012-04-14,2012-06-07,2012-05-31,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-08,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-142,0,-1,-5,-20,-20,41462,189,8,8,2,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,08-Jul-2012,Jul 8th 2012,Mon 07-08-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-08,2012-07-08,Mon ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,N,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-08,2011-07-08,2012-04-08,2012-01-08,2012-06-08,2012-05-08,2012-07-01,2012-06-24,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-30,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,3,0,0,0,0,0,41607,334,61,30,7,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,30-Nov-2012,Nov 30th 2012,Sat 11-30-13,0,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-25,2012-11-30,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,N,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-30,2011-11-30,2012-08-30,2012-05-30,2012-10-30,2012-09-30,2012-11-23,2012-11-16,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-10,2012-01-01,2012-01-01,2012-01-01,2012-01-06,0,-3,-10,-46,-321,0,-3,-11,-46,-46,41283,10,10,10,5,2,5897,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-12,10-Jan-2012,Jan 10th 2012,Thu 01-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-06,2012-01-07,2012-01-10,Thu ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,2,2012,2012-01-06,2012-01-12,N,Wk.02 - 13,2012-01-06 00:00:00,2012-01-12 00:00:00,2012W02 ,2012W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2012,N,2012-01-10,2011-01-10,2012-10-10,2012-07-10,2012-12-10,2012-11-10,2012-01-03,2012-12-27,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-18,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-193,0,-2,-7,-28,-28,41411,138,48,18,7,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,18-May-2012,May 18th 2012,Sat 05-18-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-18,Sat ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-18,2011-05-18,2012-02-18,2012-11-18,2012-04-18,2012-03-18,2012-05-11,2012-05-04,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-15,2012-01-01,2012-04-01,2012-04-01,2012-04-14,0,-2,-7,-32,-226,0,-2,-8,-32,-32,41378,105,15,15,2,16,5911,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-20,15-Apr-2012,Apr 15th 2012,Mon 04-15-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-14,2012-04-15,2012-04-15,Mon ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,16,2012,2012-04-14,2012-04-20,N,Wk.16 - 13,2012-04-14 00:00:00,2012-04-20 00:00:00,2012W16 ,2012W16 ,04/14/13 - 04/20/13,04/14 - 04/20,2012,N,2012-04-15,2011-04-15,2012-01-15,2012-10-15,2012-03-15,2012-02-15,2012-04-08,2012-04-01,0,0,0,0,0,0,0,0,4,2,16,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-12,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-260,0,-3,-9,-37,-37,41344,71,71,12,3,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,12-Mar-2012,Mar 12th 2012,Tue 03-12-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-11,2012-03-12,Tue ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,N,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-12,2011-03-12,2012-12-12,2012-09-12,2012-02-12,2012-01-12,2012-03-05,2012-02-26,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-09,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-18,0,0,-1,-3,-3,41586,313,40,9,7,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,09-Nov-2012,Nov 9th 2012,Sat 11-09-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-09,Sat ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-09,2011-11-09,2012-08-09,2012-05-09,2012-10-09,2012-09-09,2012-11-02,2012-10-26,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-14,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-258,0,-3,-9,-37,-37,41346,73,73,14,5,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,14-Mar-2012,Mar 14th 2012,Thu 03-14-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-11,2012-03-14,Thu ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,N,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-14,2011-03-14,2012-12-14,2012-09-14,2012-02-14,2012-01-14,2012-03-07,2012-02-28,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-03,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-85,0,-1,-3,-12,-12,41519,246,65,3,3,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,03-Sep-2012,Sep 3rd 2012,Tue 09-03-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-09-02,2012-09-03,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,N,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-03,2011-09-03,2012-06-03,2012-03-03,2012-08-03,2012-07-03,2012-08-27,2012-08-20,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-16,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-195,0,-2,-7,-28,-28,41409,136,46,16,5,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,16-May-2012,May 16th 2012,Thu 05-16-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-16,Thu ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-16,2011-05-16,2012-02-16,2012-11-16,2012-04-16,2012-03-16,2012-05-09,2012-05-02,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-27,2012-01-01,2012-10-01,2012-10-01,2012-10-27,0,0,-1,-4,-31,0,0,-1,-4,-5,41573,300,27,27,1,44,5939,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-11-02,27-Oct-2012,Oct 27th 2012,Sun 10-27-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-10-27,2012-10-27,2012-10-21,2012-10-27,Sun ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,44,2012,2012-10-27,2012-11-02,Y,Wk.44 - 13,2012-10-27 00:00:00,2012-11-02 00:00:00,2012W44 ,2012W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2012,N,2012-10-27,2011-10-27,2012-07-27,2012-04-27,2012-09-27,2012-08-27,2012-10-20,2012-10-13,0,0,0,0,1,0,0,0,11,4,44,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-21,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-98,0,-1,-4,-14,-14,41506,233,52,21,4,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,21-Aug-2012,Aug 21st 2012,Wed 08-21-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-21,Wed ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-21,2011-08-21,2012-05-21,2012-02-21,2012-07-21,2012-06-21,2012-08-14,2012-08-07,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-25,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-33,0,0,-2,-5,-5,41571,298,25,25,6,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,25-Oct-2012,Oct 25th 2012,Fri 10-25-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-25,Fri ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-25,2011-10-25,2012-07-25,2012-04-25,2012-09-25,2012-08-25,2012-10-18,2012-10-11,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-24,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-64,0,-1,-3,-9,-9,41540,267,86,24,3,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,24-Sep-2012,Sep 24th 2012,Tue 09-24-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-23,2012-09-24,Tue ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,N,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-24,2011-09-24,2012-06-24,2012-03-24,2012-08-24,2012-07-24,2012-09-17,2012-09-10,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-27,2012-01-01,2012-01-01,2012-01-01,2012-01-27,0,-3,-10,-43,-304,0,-3,-10,-43,-44,41300,27,27,27,1,5,5900,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-02-02,27-Jan-2012,Jan 27th 2012,Sun 01-27-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-01-27,2012-01-27,2012-01-21,2012-01-27,Sun ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,5,2012,2012-01-27,2012-02-02,Y,Wk.05 - 13,2012-01-27 00:00:00,2012-02-02 00:00:00,2012W05 ,2012W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2012,N,2012-01-27,2011-01-27,2012-10-27,2012-07-27,2012-12-27,2012-11-27,2012-01-20,2012-01-13,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-10,2012-01-01,2012-01-01,2012-03-01,2012-03-10,0,-3,-8,-37,-262,0,-3,-9,-37,-38,41342,69,69,10,1,11,5906,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-16,10-Mar-2012,Mar 10th 2012,Sun 03-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-10,2012-03-04,2012-03-10,Sun ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,11,2012,2012-03-10,2012-03-16,Y,Wk.11 - 13,2012-03-10 00:00:00,2012-03-16 00:00:00,2012W11 ,2012W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2012,N,2012-03-10,2011-03-10,2012-12-10,2012-09-10,2012-02-10,2012-01-10,2012-03-03,2012-02-24,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-14,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-197,0,-2,-7,-28,-28,41407,134,44,14,3,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,14-May-2012,May 14th 2012,Tue 05-14-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-13,2012-05-14,Tue ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,N,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-14,2011-05-14,2012-02-14,2012-11-14,2012-04-14,2012-03-14,2012-05-07,2012-04-30,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-31,2012-01-01,2012-10-01,2012-12-01,2012-12-29,0,0,1,5,34,1,1,1,5,5,41638,365,92,31,3,53,5948,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-31,31-Dec-2012,Dec 31st 2012,Tue 12-31-13,0,0,0,0,2012-12-21,365,92,31,3,2012-12-29,2012-12-29,2012-12-29,2012-12-29,2012-12-30,2012-12-31,Tue ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,1,2014,2012-12-29,2014-01-04,N,Wk.53 - 13,2012-12-29 00:00:00,2012-12-31 00:00:00,2012W53 ,2012W53 ,12/29/13 - 12/31/13,12/29 - 12/31,2012,N,2012-12-31,2011-12-31,2012-09-30,2012-06-30,2012-11-30,2012-10-31,2012-12-24,2012-12-17,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-01,2012-01-01,2012-07-01,2012-09-01,2012-09-01,0,-1,-2,-12,-87,0,-1,-3,-12,-13,41517,244,63,1,1,36,5931,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-07,01-Sep-2012,Sep 1st 2012,Sun 09-01-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-01,2012-08-26,2012-09-01,Sun ,2012M09,Sep-2012,Y,2012M09 ,Y,Year 2012 - Quarter 03,2012Q03 ,N,36,2012,2012-09-01,2012-09-07,Y,Wk.36 - 13,2012-09-01 00:00:00,2012-09-07 00:00:00,2012W36 ,2012W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2012,N,2012-09-01,2011-09-01,2012-06-01,2012-03-01,2012-08-01,2012-07-01,2012-08-25,2012-08-18,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-13,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-228,0,-2,-8,-33,-33,41376,103,13,13,7,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,13-Apr-2012,Apr 13th 2012,Sat 04-13-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-13,Sat ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-13,2011-04-13,2012-01-13,2012-10-13,2012-03-13,2012-02-13,2012-04-06,2012-03-30,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-09,2012-01-01,2012-01-01,2012-02-01,2012-02-03,0,-3,-9,-42,-291,0,-3,-10,-42,-42,41313,40,40,9,7,6,5901,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-02-09,09-Feb-2012,Feb 9th 2012,Sat 02-09-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-01-27,2012-02-03,2012-02-04,2012-02-09,Sat ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,6,2012,2012-02-03,2012-02-09,N,Wk.06 - 13,2012-02-03 00:00:00,2012-02-09 00:00:00,2012W06 ,2012W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2012,N,2012-02-09,2011-02-09,2012-11-09,2012-08-09,2012-01-09,2012-12-09,2012-02-02,2012-01-26,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-27,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,30,0,0,0,4,4,41634,361,88,27,6,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,27-Dec-2012,Dec 27th 2012,Fri 12-27-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-27,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-27,2011-12-27,2012-09-27,2012-06-27,2012-11-27,2012-10-27,2012-12-20,2012-12-13,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-19,2012-01-01,2012-07-01,2012-08-01,2012-08-18,0,-1,-3,-14,-100,0,-1,-4,-14,-14,41504,231,50,19,2,34,5929,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-24,19-Aug-2012,Aug 19th 2012,Mon 08-19-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-18,2012-08-19,2012-08-19,Mon ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,34,2012,2012-08-18,2012-08-24,N,Wk.34 - 13,2012-08-18 00:00:00,2012-08-24 00:00:00,2012W34 ,2012W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2012,N,2012-08-19,2011-08-19,2012-05-19,2012-02-19,2012-07-19,2012-06-19,2012-08-12,2012-08-05,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-04-11,2012-01-01,2012-04-01,2012-04-01,2012-04-07,0,-2,-7,-33,-230,0,-2,-8,-33,-33,41374,101,11,11,5,15,5910,1,4,1360,2,454,114,2012-12-31,2012-06-30,2012-04-30,2012-04-13,11-Apr-2012,Apr 11th 2012,Thu 04-11-13,1,0,0,0,2012-03-21,365,91,30,7,2012-12-30,2012-03-31,2012-03-31,2012-04-07,2012-04-08,2012-04-11,Thu ,2012M04,Apr-2012,N,2012M04 ,N,Year 2012 - Quarter 02,2012Q02 ,N,15,2012,2012-04-07,2012-04-13,N,Wk.15 - 13,2012-04-07 00:00:00,2012-04-13 00:00:00,2012W15 ,2012W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2012,N,2012-04-11,2011-04-11,2012-01-11,2012-10-11,2012-03-11,2012-02-11,2012-04-04,2012-03-28,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-06,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,9,0,0,0,1,1,41613,340,67,6,6,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,06-Dec-2012,Dec 6th 2012,Fri 12-06-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-06,Fri ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-06,2011-12-06,2012-09-06,2012-06-06,2012-11-06,2012-10-06,2012-11-29,2012-11-22,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-31,2012-01-01,2012-01-01,2012-03-01,2012-03-31,0,-3,-8,-34,-241,0,-2,-8,-34,-35,41363,90,90,31,1,14,5909,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-04-06,31-Mar-2012,Mar 31st 2012,Sun 03-31-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-03-31,2012-03-31,2012-03-31,2012-03-25,2012-03-31,Sun ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,14,2012,2012-03-31,2012-04-06,Y,Wk.14 - 13,2012-03-31 00:00:00,2012-04-06 00:00:00,2012W14 ,2012W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2012,N,2012-03-31,2011-03-31,2012-12-31,2012-09-30,2012-02-28,2012-01-31,2012-03-24,2012-03-17,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-22,2012-01-01,2012-07-01,2012-09-01,2012-09-22,0,-1,-2,-9,-66,0,-1,-3,-9,-10,41538,265,84,22,1,39,5934,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-28,22-Sep-2012,Sep 22nd 2012,Sun 09-22-13,1,0,0,0,2012-09-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-22,2012-09-16,2012-09-22,Sun ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,39,2012,2012-09-22,2012-09-28,Y,Wk.39 - 13,2012-09-22 00:00:00,2012-09-28 00:00:00,2012W39 ,2012W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2012,N,2012-09-22,2011-09-22,2012-06-22,2012-03-22,2012-08-22,2012-07-22,2012-09-15,2012-09-08,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-06,2012-01-01,2012-01-01,2012-01-01,2012-01-06,0,-3,-10,-46,-325,0,-3,-11,-46,-47,41279,6,6,6,1,2,5897,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-12,06-Jan-2012,Jan 6th 2012,Sun 01-06-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2012-01-06,2012-12-31,2012-01-06,Sun ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,2,2012,2012-01-06,2012-01-12,Y,Wk.02 - 13,2012-01-06 00:00:00,2012-01-12 00:00:00,2012W02 ,2012W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2012,N,2012-01-06,2011-01-06,2012-10-06,2012-07-06,2012-12-06,2012-11-06,2012-12-30,2012-12-23,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-04,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-327,0,-3,-11,-47,-47,41277,4,4,4,6,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,04-Jan-2012,Jan 4th 2012,Fri 01-04-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-04,Fri ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-04,2011-01-04,2012-10-04,2012-07-04,2012-12-04,2012-11-04,2012-12-28,2012-12-21,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-16,2012-01-01,2012-07-01,2012-07-01,2012-07-14,0,-1,-4,-19,-134,0,-1,-5,-19,-19,41470,197,16,16,3,29,5924,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-20,16-Jul-2012,Jul 16th 2012,Tue 07-16-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-14,2012-07-15,2012-07-16,Tue ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,29,2012,2012-07-14,2012-07-20,N,Wk.29 - 13,2012-07-14 00:00:00,2012-07-20 00:00:00,2012W29 ,2012W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2012,N,2012-07-16,2011-07-16,2012-04-16,2012-01-16,2012-06-16,2012-05-16,2012-07-09,2012-07-02,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-03,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-24,0,0,-1,-3,-4,41580,307,34,3,1,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,03-Nov-2012,Nov 3rd 2012,Sun 11-03-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-10-28,2012-11-03,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,Y,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-03,2011-11-03,2012-08-03,2012-05-03,2012-10-03,2012-09-03,2012-10-27,2012-10-20,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-25,2012-01-01,2012-07-01,2012-08-01,2012-08-25,0,-1,-3,-13,-94,0,-1,-3,-13,-14,41510,237,56,25,1,35,5930,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-31,25-Aug-2012,Aug 25th 2012,Sun 08-25-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-08-25,2012-08-25,2012-08-19,2012-08-25,Sun ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,35,2012,2012-08-25,2012-08-31,Y,Wk.35 - 13,2012-08-25 00:00:00,2012-08-31 00:00:00,2012W35 ,2012W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2012,N,2012-08-25,2011-08-25,2012-05-25,2012-02-25,2012-07-25,2012-06-25,2012-08-18,2012-08-11,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-24,2012-01-01,2012-10-01,2012-11-01,2012-11-24,0,0,0,0,-3,0,0,0,0,-1,41601,328,55,24,1,48,5943,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-30,24-Nov-2012,Nov 24th 2012,Sun 11-24-13,1,1,1,1,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-11-24,2012-11-24,2012-11-18,2012-11-24,Sun ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,48,2012,2012-11-24,2012-11-30,Y,Wk.48 - 13,2012-11-24 00:00:00,2012-11-30 00:00:00,2012W48 ,2012W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2012,N,2012-11-24,2011-11-24,2012-08-24,2012-05-24,2012-10-24,2012-09-24,2012-11-17,2012-11-10,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-12,2012-01-01,2012-04-01,2012-05-01,2012-05-12,0,-2,-6,-28,-199,0,-2,-7,-28,-29,41405,132,42,12,1,20,5915,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-18,12-May-2012,May 12th 2012,Sun 05-12-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-12,2012-05-06,2012-05-12,Sun ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,20,2012,2012-05-12,2012-05-18,Y,Wk.20 - 13,2012-05-12 00:00:00,2012-05-18 00:00:00,2012W20 ,2012W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2012,N,2012-05-12,2011-05-12,2012-02-12,2012-11-12,2012-04-12,2012-03-12,2012-05-05,2012-04-28,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-08-17,2012-01-01,2012-07-01,2012-08-01,2012-08-11,0,-1,-3,-15,-102,0,-1,-4,-15,-15,41502,229,48,17,7,33,5928,2,8,1364,3,455,114,2012-12-31,2012-09-30,2012-08-31,2012-08-17,17-Aug-2012,Aug 17th 2012,Sat 08-17-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-07-28,2012-08-11,2012-08-12,2012-08-17,Sat ,2012M08,Aug-2012,N,2012M08 ,N,Year 2012 - Quarter 03,2012Q03 ,N,33,2012,2012-08-11,2012-08-17,N,Wk.33 - 13,2012-08-11 00:00:00,2012-08-17 00:00:00,2012W33 ,2012W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2012,N,2012-08-17,2011-08-17,2012-05-17,2012-02-17,2012-07-17,2012-06-17,2012-08-10,2012-08-03,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-21,2012-01-01,2012-10-01,2012-10-01,2012-10-20,0,0,-1,-5,-37,0,0,-2,-5,-5,41567,294,21,21,2,43,5938,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-26,21-Oct-2012,Oct 21st 2012,Mon 10-21-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-20,2012-10-21,2012-10-21,Mon ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,43,2012,2012-10-20,2012-10-26,N,Wk.43 - 13,2012-10-20 00:00:00,2012-10-26 00:00:00,2012W43 ,2012W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2012,N,2012-10-21,2011-10-21,2012-07-21,2012-04-21,2012-09-21,2012-08-21,2012-10-14,2012-10-07,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-03-27,2012-01-01,2012-01-01,2012-03-01,2012-03-24,0,-3,-8,-35,-245,0,-3,-9,-35,-35,41359,86,86,27,4,13,5908,3,3,1359,1,453,114,2012-12-31,2012-03-31,2012-03-31,2012-03-30,27-Mar-2012,Mar 27th 2012,Wed 03-27-13,1,0,0,0,2012-03-21,365,90,31,7,2012-12-30,2012-12-30,2012-02-24,2012-03-24,2012-03-25,2012-03-27,Wed ,2012M03,Mar-2012,N,2012M03 ,N,Year 2012 - Quarter 01,2012Q01 ,N,13,2012,2012-03-24,2012-03-30,N,Wk.13 - 13,2012-03-24 00:00:00,2012-03-30 00:00:00,2012W13 ,2012W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2012,N,2012-03-27,2011-03-27,2012-12-27,2012-09-27,2012-02-27,2012-01-27,2012-03-20,2012-03-13,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-22,2012-01-01,2012-10-01,2012-11-01,2012-11-17,0,0,0,-1,-5,0,0,-1,-1,-1,41599,326,53,22,6,47,5942,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-23,22-Nov-2012,Nov 22nd 2012,Fri 11-22-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-17,2012-11-18,2012-11-22,Fri ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,47,2012,2012-11-17,2012-11-23,N,Wk.47 - 13,2012-11-17 00:00:00,2012-11-23 00:00:00,2012W47 ,2012W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2012,N,2012-11-22,2011-11-22,2012-08-22,2012-05-22,2012-10-22,2012-09-22,2012-11-15,2012-11-08,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-07-07,2012-01-01,2012-07-01,2012-07-01,2012-07-07,0,-1,-4,-20,-143,0,-1,-5,-20,-21,41461,188,7,7,1,28,5923,1,7,1363,3,455,114,2012-12-31,2012-09-30,2012-07-31,2012-07-13,07-Jul-2012,Jul 7th 2012,Sun 07-07-13,1,0,0,0,2012-06-21,365,92,31,7,2012-12-30,2012-06-30,2012-06-30,2012-07-07,2012-07-01,2012-07-07,Sun ,2012M07,Jul-2012,N,2012M07 ,N,Year 2012 - Quarter 03,2012Q03 ,N,28,2012,2012-07-07,2012-07-13,Y,Wk.28 - 13,2012-07-07 00:00:00,2012-07-13 00:00:00,2012W28 ,2012W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2012,N,2012-07-07,2011-07-07,2012-04-07,2012-01-07,2012-06-07,2012-05-07,2012-06-30,2012-06-23,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-09-11,2012-01-01,2012-07-01,2012-09-01,2012-09-08,0,-1,-2,-11,-77,0,-1,-3,-11,-11,41527,254,73,11,4,37,5932,3,9,1365,3,455,114,2012-12-31,2012-09-30,2012-09-30,2012-09-14,11-Sep-2012,Sep 11th 2012,Wed 09-11-13,1,0,0,0,2012-06-21,365,92,30,7,2012-12-30,2012-06-30,2012-08-25,2012-09-08,2012-09-09,2012-09-11,Wed ,2012M09,Sep-2012,N,2012M09 ,N,Year 2012 - Quarter 03,2012Q03 ,N,37,2012,2012-09-08,2012-09-14,N,Wk.37 - 13,2012-09-08 00:00:00,2012-09-14 00:00:00,2012W37 ,2012W37 ,09/08/13 - 09/14/13,09/08 - 09/14,2012,N,2012-09-11,2011-09-11,2012-06-11,2012-03-11,2012-08-11,2012-07-11,2012-09-04,2012-08-28,0,0,0,0,0,0,0,0,9,3,37,3,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-10-19,2012-01-01,2012-10-01,2012-10-01,2012-10-13,0,0,-1,-6,-39,0,0,-2,-6,-6,41565,292,19,19,7,42,5937,1,10,1366,4,456,114,2012-12-31,2012-12-31,2012-10-31,2012-10-19,19-Oct-2012,Oct 19th 2012,Sat 10-19-13,1,1,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-09-29,2012-10-13,2012-10-14,2012-10-19,Sat ,2012M10,Oct-2012,N,2012M10 ,N,Year 2012 - Quarter 04,2012Q04 ,N,42,2012,2012-10-13,2012-10-19,N,Wk.42 - 13,2012-10-13 00:00:00,2012-10-19 00:00:00,2012W42 ,2012W42 ,10/13/13 - 10/19/13,10/13 - 10/19,2012,N,2012-10-19,2011-10-19,2012-07-19,2012-04-19,2012-09-19,2012-08-19,2012-10-12,2012-10-05,0,0,0,0,0,0,0,0,10,4,42,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-23,2012-01-01,2012-10-01,2012-12-01,2012-12-22,0,0,1,4,26,0,0,0,4,4,41630,357,84,23,2,52,5947,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-28,23-Dec-2012,Dec 23rd 2012,Mon 12-23-13,0,0,0,0,2012-12-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-22,2012-12-23,2012-12-23,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,52,2012,2012-12-22,2012-12-28,N,Wk.52 - 13,2012-12-22 00:00:00,2012-12-28 00:00:00,2012W52 ,2012W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2012,N,2012-12-23,2011-12-23,2012-09-23,2012-06-23,2012-11-23,2012-10-23,2012-12-16,2012-12-09,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-24,2012-01-01,2012-04-01,2012-05-01,2012-05-19,0,-2,-6,-27,-187,0,-2,-7,-27,-27,41417,144,54,24,6,21,5916,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-05-25,24-May-2012,May 24th 2012,Fri 05-24-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-04-28,2012-05-19,2012-05-20,2012-05-24,Fri ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,21,2012,2012-05-19,2012-05-25,N,Wk.21 - 13,2012-05-19 00:00:00,2012-05-25 00:00:00,2012W21 ,2012W21 ,05/19/13 - 05/25/13,05/19 - 05/25,2012,N,2012-05-24,2011-05-24,2012-02-24,2012-11-24,2012-04-24,2012-03-24,2012-05-17,2012-05-10,0,0,0,0,0,0,0,0,5,2,21,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-11-05,2012-01-01,2012-10-01,2012-11-01,2012-11-03,0,0,0,-3,-22,0,0,-1,-3,-3,41582,309,36,5,3,45,5940,2,11,1367,4,456,114,2012-12-31,2012-12-31,2012-11-30,2012-11-09,05-Nov-2012,Nov 5th 2012,Tue 11-05-13,1,1,1,0,2012-09-21,365,92,30,7,2012-12-30,2012-09-29,2012-10-27,2012-11-03,2012-11-04,2012-11-05,Tue ,2012M11,Nov-2012,N,2012M11 ,N,Year 2012 - Quarter 04,2012Q04 ,N,45,2012,2012-11-03,2012-11-09,N,Wk.45 - 13,2012-11-03 00:00:00,2012-11-09 00:00:00,2012W45 ,2012W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2012,N,2012-11-05,2011-11-05,2012-08-05,2012-05-05,2012-10-05,2012-09-05,2012-10-29,2012-10-22,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-05-29,2012-01-01,2012-04-01,2012-05-01,2012-05-26,0,-2,-6,-26,-182,0,-2,-6,-26,-26,41422,149,59,29,4,22,5917,2,5,1361,2,454,114,2012-12-31,2012-06-30,2012-05-31,2012-06-01,29-May-2012,May 29th 2012,Wed 05-29-13,1,0,0,0,2012-03-21,365,91,31,7,2012-12-30,2012-03-31,2012-05-26,2012-05-26,2012-05-27,2012-05-29,Wed ,2012M05,May-2012,N,2012M05 ,N,Year 2012 - Quarter 02,2012Q02 ,N,22,2012,2012-05-26,2012-06-01,N,Wk.22 - 13,2012-05-26 00:00:00,2012-06-01 00:00:00,2012W22 ,2012W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2012,N,2012-05-29,2011-05-29,2012-02-28,2012-11-29,2012-04-29,2012-03-29,2012-05-22,2012-05-15,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-01-02,2012-01-01,2012-01-01,2012-01-01,2012-01-01,0,-3,-10,-47,-329,0,-3,-11,-47,-47,41275,2,2,2,4,1,5896,1,1,1357,1,453,114,2012-12-31,2012-03-31,2012-01-31,2012-01-05,02-Jan-2012,Jan 2nd 2012,Wed 01-02-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2012-01-02,Wed ,2012M01,Jan-2012,N,2012M01 ,N,Year 2012 - Quarter 01,2012Q01 ,N,1,2012,2012-12-30,2012-01-05,N,Wk.01 - 13,2012-01-01 00:00:00,2012-01-05 00:00:00,2012W01 ,2012W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2012,N,2012-01-02,2011-01-02,2012-10-02,2012-07-02,2012-12-02,2012-11-02,2012-12-26,2012-12-19,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-12-02,2012-01-01,2012-10-01,2012-12-01,2012-12-01,0,0,1,1,5,0,0,0,1,1,41609,336,63,2,2,49,5944,3,12,1368,4,456,114,2012-12-31,2012-12-31,2012-12-31,2012-12-07,02-Dec-2012,Dec 2nd 2012,Mon 12-02-13,0,0,0,0,2012-09-21,365,92,31,7,2012-12-30,2012-09-29,2012-11-24,2012-12-01,2012-12-02,2012-12-02,Mon ,2012M12,Dec-2012,N,2012M12 ,N,Year 2012 - Quarter 04,2012Q04 ,N,49,2012,2012-12-01,2012-12-07,N,Wk.49 - 13,2012-12-01 00:00:00,2012-12-07 00:00:00,2012W49 ,2012W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2012,N,2012-12-02,2011-12-02,2012-09-02,2012-06-02,2012-11-02,2012-10-02,2012-11-25,2012-11-18,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,USER_X ,2012-11-27 00:16:56,USER_X
-2012-02-24,2012-01-01,2012-01-01,2012-02-01,2012-02-24,0,-3,-9,-39,-276,0,-3,-9,-39,-40,41328,55,55,24,1,9,5904,2,2,1358,1,453,114,2012-12-31,2012-03-31,2012-02-28,2012-03-02,24-Feb-2012,Feb 24th 2012,Sun 02-24-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2012-02-24,2012-02-24,2012-02-18,2012-02-24,Sun ,2012M02,Feb-2012,N,2012M02 ,N,Year 2012 - Quarter 01,2012Q01 ,N,9,2012,2012-02-24,2012-03-02,Y,Wk.09 - 13,2012-02-24 00:00:00,2012-03-02 00:00:00,2012W09 ,2012W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2012,N,2
<TRUNCATED>
[18/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
deleted file mode 100644
index 3a7db5e..0000000
--- a/examples/test_case_data/sandbox/kylin.properties
+++ /dev/null
@@ -1,144 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=localhost:7070
-
-#set display timezone on UI,format like[GMT+N or GMT-N]
-kylin.rest.timezone=GMT-8
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-# Hive database name for putting the intermediate flat tables
-kylin.job.hive.database.for.intermediatetable=default
-
-#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=gzip
-
-# The cut size for hbase region, in GB.
-# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
-kylin.hbase.region.cut.small=10
-kylin.hbase.region.cut.medium=20
-kylin.hbase.region.cut.large=100
-
-# HBase min and max region count
-kylin.hbase.region.count.min=1
-kylin.hbase.region.count.max=500
-
-## Config for Restful APP ##
-
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-# route to hive settings
-kylin.route.hive.enabled=false
-# default to the embedded server (jdbc:hive2://)
-#kylin.route.hive.url=
-#kylin.route.hive.username=
-#kylin.route.hive.password=
-
-## Config for mail service
-
-# If true, will send email notification;
-mail.enabled=false
-mail.host=
-mail.username=
-mail.password=
-mail.sender=
-
-###########################config info for web#######################
-
-#help info ,format{name|displayName|link} ,optional
-kylin.web.help.length=4
-kylin.web.help.0=start|Getting Started|
-kylin.web.help.1=odbc|ODBC Driver|
-kylin.web.help.2=tableau|Tableau Guide|
-kylin.web.help.3=onboard|Cube Design Tutorial|
-#hadoop url link ,optional
-kylin.web.hadoop=
-#job diagnostic url link ,optional
-kylin.web.diagnostic=
-#contact mail on web page ,optional
-kylin.web.contact_mail=
-
-#set the number of databases and tables of each database shown on the hive tree
-kylin.web.hive.limit=20
-
-###########################config info for front#######################
-
-#env DEV|QA|PROD
-deploy.env=DEV
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
-
-
-###########################config info for kylin monitor#######################
-# hive jdbc url
-kylin.monitor.hive.jdbc.connection.url= jdbc:hive2://sandbox:10000
-
-#config where to parse query log,split with comma ,will also read $KYLIN_HOME/tomcat/logs/ by default
-kylin.monitor.ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2
-
-#will create external hive table to query result csv file
-#will set to kylin_query_log by default if not config here
-kylin.monitor.query.log.parse.result.table = kylin_query_log
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
deleted file mode 100644
index 81704f2..0000000
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
deleted file mode 100644
index a8ff87f..0000000
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ /dev/null
@@ -1,254 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>io.sort.mb</name>
- <value>64</value>
- </property>
-
- <property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
- </property>
-
- <property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- </property>
-
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
- <property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>64</value>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
deleted file mode 100644
index 8256158..0000000
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ /dev/null
@@ -1,534 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
- <property>
- <name>yarn.acl.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
- </property>
-
- <property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/invertedindex/.settings/org.eclipse.core.resources.prefs b/invertedindex/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 839d647..0000000
--- a/invertedindex/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/invertedindex/.settings/org.eclipse.jdt.core.prefs b/invertedindex/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 7c91e8d..0000000
--- a/invertedindex/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/invertedindex/.settings/org.eclipse.jdt.ui.prefs b/invertedindex/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index 681b0b0..0000000
--- a/invertedindex/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
deleted file mode 100644
index 7fa3e52..0000000
--- a/invertedindex/pom.xml
+++ /dev/null
@@ -1,165 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>kylin</artifactId>
- <groupId>org.apache.kylin</groupId>
- <version>1.3-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-invertedindex</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:InvertedIndex</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.parent.version}</version>
- </dependency>
-
- <!--Kylin Jar -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-metadata</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-dictionary</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.ning</groupId>
- <artifactId>compress-lzf</artifactId>
- </dependency>
- <dependency>
- <groupId>org.roaringbitmap</groupId>
- <artifactId>RoaringBitmap</artifactId>
- </dependency>
- <!--<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
- </dependency>-->
-
- <!-- Env & Test -->
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <classifier>hadoop2</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- <!-- version conflict with hadoop2.2 -->
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
deleted file mode 100644
index 4ee8f50..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Sets;
-
-/**
- */
-public class IICapabilityChecker {
- private static final Logger logger = LoggerFactory.getLogger(IICapabilityChecker.class);
-
- public static CapabilityResult check(IIInstance ii, SQLDigest digest) {
- CapabilityResult result = new CapabilityResult();
- result.capable = false;
-
- // match fact table
- if (!digest.factTable.equalsIgnoreCase(ii.getFactTable())) {
- logger.info("Exclude II " + ii.getName() + " because fact table unmatch");
- return result;
- }
-
- // match joins
- boolean matchJoin = isMatchedWithJoins(digest.joinDescs, ii);
- if (!matchJoin) {
- logger.info("Exclude II " + ii.getName() + " because unmatched joins");
- return result;
- }
-
- // dimensions & measures
- Collection<TblColRef> dimensionColumns = getDimensionColumns(digest);
- Collection<FunctionDesc> aggrFunctions = digest.aggregations;
- Collection<TblColRef> unmatchedDimensions = unmatchedDimensions(dimensionColumns, ii);
- Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, ii);
-
- // try dimension-as-measure
- if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, ii, result);
- }
-
- if (!unmatchedDimensions.isEmpty()) {
- logger.info("Exclude ii " + ii.getName() + " because unmatched dimensions");
- return result;
- }
-
- if (!unmatchedAggregations.isEmpty()) {
- logger.info("Exclude ii " + ii.getName() + " because unmatched aggregations");
- return result;
- }
-
- // cost will be minded by caller
- result.capable = true;
- return result;
- }
-
- private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, IIInstance iiInstance) {
- IIDesc iiDesc = iiInstance.getDescriptor();
- List<TableDesc> tables = iiDesc.listTables();
-
- List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(tables.size());
- for (TableDesc tableDesc : tables) {
- JoinDesc join = null;
- for (LookupDesc lookup : iiDesc.getModel().getLookups()) {
- if (lookup.getTable().equalsIgnoreCase(tableDesc.getIdentity())) {
- join = lookup.getJoin();
- cubeJoins.add(join);
- break;
- }
- }
- }
-
- for (JoinDesc j : joins) {
- // optiq engine can't decide which one is fk or pk
- String pTable = j.getPrimaryKeyColumns()[0].getTable();
- String factTable = iiDesc.getModel().getFactTable();
- if (factTable.equals(pTable)) {
- j.swapPKFK();
- }
-
- // check primary key, all PK column should refer to same tale, the Fact Table of iiInstance.
- // Using first column's table name to check.
- String fTable = j.getForeignKeyColumns()[0].getTable();
- if (!factTable.equals(fTable)) {
- logger.info("Fact Table" + factTable + " not matched in join: " + j + " on ii " + iiInstance.getName());
- return false;
- }
-
- // The hashcode() function of JoinDesc has been overwritten,
- // which takes into consideration: pk,fk,jointype
- if (!cubeJoins.contains(j)) {
- logger.info("Query joins don't match on ii " + iiInstance.getName());
- return false;
- }
- }
- return true;
- }
-
- private static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) {
- Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns;
- Collection<TblColRef> filterColumns = sqlDigest.filterColumns;
-
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-
- private static Set<TblColRef> unmatchedDimensions(Collection<TblColRef> dimensionColumns, IIInstance ii) {
- HashSet<TblColRef> result = Sets.newHashSet(dimensionColumns);
- result.removeAll(ii.getDescriptor().listAllDimensions());
- return result;
- }
-
- private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, IIInstance ii) {
- HashSet<FunctionDesc> result = Sets.newHashSet(aggregations);
- result.removeAll(ii.getDescriptor().listAllFunctions());
- return result;
- }
-
- private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, IIInstance ii, CapabilityResult result) {
- IIDesc iiDesc = ii.getDescriptor();
- Collection<FunctionDesc> iiFuncs = iiDesc.listAllFunctions();
-
- Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
- while (it.hasNext()) {
- FunctionDesc functionDesc = it.next();
-
- if (iiFuncs.contains(functionDesc)) {
- it.remove();
- continue;
- }
-
- // let calcite handle count
- if (functionDesc.isCount()) {
- it.remove();
- continue;
- }
-
- // calcite can do aggregation from columns on-the-fly
- List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
- if (neededCols.size() > 0 && iiDesc.listAllDimensions().containsAll(neededCols)) {
- result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
- it.remove();
- continue;
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
deleted file mode 100644
index 804ecbc..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * copied from CubeDescManager
- *
- * @author honma
- */
-public class IIDescManager {
-
- private static final Logger logger = LoggerFactory.getLogger(IIDescManager.class);
-
- private static final Serializer<IIDesc> II_DESC_SERIALIZER = new JsonSerializer<IIDesc>(IIDesc.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, IIDescManager> CACHE = new ConcurrentHashMap<KylinConfig, IIDescManager>();
-
- // ============================================================================
-
- private KylinConfig config;
- // name ==> IIDesc
- private CaseInsensitiveStringCache<IIDesc> iiDescMap = new CaseInsensitiveStringCache<IIDesc>(Broadcaster.TYPE.INVERTED_INDEX_DESC);
-
- public static IIDescManager getInstance(KylinConfig config) {
- IIDescManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (IIDescManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new IIDescManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init IIDescManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- private IIDescManager(KylinConfig config) throws IOException {
- logger.info("Initializing IIDescManager with config " + config);
- this.config = config;
- reloadAllIIDesc();
- }
-
- public IIDesc getIIDesc(String name) {
- return iiDescMap.get(name);
- }
-
- /**
- * Reload IIDesc from resource store It will be triggered by an desc update
- * event.
- *
- * @param name
- * @throws IOException
- */
- public IIDesc reloadIIDesc(String name) throws IOException {
-
- // Save Source
- String path = IIDesc.getIIDescResourcePath(name);
-
- // Reload the IIDesc
- IIDesc ndesc = loadIIDesc(path);
-
- // Here replace the old one
- iiDescMap.putLocal(ndesc.getName(), ndesc);
- return ndesc;
- }
-
- private IIDesc loadIIDesc(String path) throws IOException {
- ResourceStore store = getStore();
- logger.info("Loading IIDesc " + store.getReadableResourcePath(path));
-
- IIDesc ndesc = store.getResource(path, IIDesc.class, II_DESC_SERIALIZER);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("IIDesc name must not be blank");
- }
-
- ndesc.init(getMetadataManager());
-
- return ndesc;
- }
-
- /**
- * Create a new IIDesc
- *
- * @param iiDesc
- * @return
- * @throws IOException
- */
- public IIDesc createIIDesc(IIDesc iiDesc) throws IOException {
- if (iiDesc.getUuid() == null || iiDesc.getName() == null)
- throw new IllegalArgumentException();
-
- if (iiDescMap.containsKey(iiDesc.getName()))
- throw new IllegalArgumentException("IIDesc '" + iiDesc.getName() + "' already exists");
-
- iiDesc.init(getMetadataManager());
-
- // Check base validation
- // Semantic validation
- // TODO
-
- iiDesc.setSignature(iiDesc.calculateSignature());
-
- String path = iiDesc.getResourcePath();
- getStore().putResource(path, iiDesc, II_DESC_SERIALIZER);
- iiDescMap.put(iiDesc.getName(), iiDesc);
-
- return iiDesc;
- }
-
- // remove iiDesc
- public void removeIIDesc(IIDesc iiDesc) throws IOException {
- String path = iiDesc.getResourcePath();
- getStore().deleteResource(path);
- iiDescMap.remove(iiDesc.getName());
- }
-
- public void removeIIDescLocal(String name) throws IOException {
- iiDescMap.remove(name);
- }
-
- private void reloadAllIIDesc() throws IOException {
- ResourceStore store = getStore();
- logger.info("Reloading all II desc from folder " + store.getReadableResourcePath(ResourceStore.II_DESC_RESOURCE_ROOT));
-
- iiDescMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.II_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- logger.info("loading II Desc from path" + path);
- IIDesc desc;
- try {
- desc = loadIIDesc(path);
- } catch (Exception e) {
- logger.error("Error loading II desc " + path, e);
- continue;
- }
- if (!path.equals(desc.getResourcePath())) {
- logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath());
- continue;
- }
- if (iiDescMap.containsKey(desc.getName())) {
- logger.error("Dup IIDesc name '" + desc.getName() + "' on path " + path);
- continue;
- }
-
- iiDescMap.putLocal(desc.getName(), desc);
- }
-
- logger.debug("Loaded " + iiDescMap.size() + " II desc(s)");
- }
-
- /**
- * Update IIDesc with the input. Broadcast the event into cluster
- *
- * @param desc
- * @return
- * @throws IOException
- */
- public IIDesc updateIIDesc(IIDesc desc) throws IOException {
- // Validate IIDesc
- if (desc.getUuid() == null || desc.getName() == null) {
- throw new IllegalArgumentException();
- }
- String name = desc.getName();
- if (!iiDescMap.containsKey(name)) {
- throw new IllegalArgumentException("IIDesc '" + name + "' does not exist.");
- }
-
- desc.init(getMetadataManager());
-
- // TODO: Semantic validation
-
- desc.setSignature(desc.calculateSignature());
-
- // Save Source
- String path = desc.getResourcePath();
- getStore().putResource(path, desc, II_DESC_SERIALIZER);
-
- // Reload the IIDesc
- IIDesc ndesc = loadIIDesc(path);
- // Here replace the old one
- iiDescMap.put(ndesc.getName(), desc);
-
- return ndesc;
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
-}
[37/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
deleted file mode 100644
index 41d75c2..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeDesc.java
+++ /dev/null
@@ -1,831 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.net.util.Base64;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeDesc extends RootPersistentEntity {
-
- public static enum CubeCapacity {
- SMALL, MEDIUM, LARGE;
- }
-
- public static enum DeriveType {
- LOOKUP, PK_FK
- }
-
- public static class DeriveInfo {
- public DeriveType type;
- public DimensionDesc dimension;
- public TblColRef[] columns;
- public boolean isOneToOne; // only used when ref from derived to host
-
- DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
- this.type = type;
- this.dimension = dimension;
- this.columns = columns;
- this.isOneToOne = isOneToOne;
- }
-
- @Override
- public String toString() {
- return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
- }
-
- }
-
- private KylinConfig config;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("description")
- private String description;
- @JsonProperty("fact_table")
- private String factTable;
- @JsonProperty("null_string")
- private String[] nullStrings;
- @JsonProperty("filter_condition")
- private String filterCondition;
- @JsonProperty("cube_partition_desc")
- CubePartitionDesc cubePartitionDesc;
- @JsonProperty("dimensions")
- private List<DimensionDesc> dimensions;
- @JsonProperty("measures")
- private List<MeasureDesc> measures;
- @JsonProperty("rowkey")
- private RowKeyDesc rowkey;
- @JsonProperty("hbase_mapping")
- private HBaseMappingDesc hbaseMapping;
- @JsonProperty("signature")
- private String signature;
- @JsonProperty("capacity")
- private CubeCapacity capacity = CubeCapacity.MEDIUM;
- @JsonProperty("notify_list")
- private List<String> notifyList;
-
- private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
- private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
- private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
-
- /**
- * Error messages during resolving json metadata
- */
- private List<String> errors = new ArrayList<String>();
-
- /**
- * @return all columns this cube can support, including derived
- */
- public Set<TblColRef> listAllColumns() {
- return allColumns;
- }
-
- /**
- * @return dimension columns including derived, BUT NOT measures
- */
- public Set<TblColRef> listDimensionColumnsIncludingDerived() {
- return dimensionColumns;
- }
-
- /**
- * @return dimension columns excluding derived and measures
- */
- public List<TblColRef> listDimensionColumnsExcludingDerived() {
- List<TblColRef> result = new ArrayList<TblColRef>();
- for (TblColRef col : dimensionColumns) {
- if (isDerived(col) == false)
- result.add(col);
- }
- return result;
- }
-
- /**
- * Find FunctionDesc by Full Expression.
- *
- * @return
- */
- public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
- for (MeasureDesc m : measures) {
- if (m.getFunction().equals(manualFunc))
- return m.getFunction();
- }
- return null;
- }
-
- public TblColRef findColumnRef(String table, String column) {
- Map<String, TblColRef> cols = columnMap.get(table);
- if (cols == null)
- return null;
- else
- return cols.get(column);
- }
-
- public DimensionDesc findDimensionByColumn(TblColRef col) {
- for (DimensionDesc dim : dimensions) {
- if (ArrayUtils.contains(dim.getColumnRefs(), col))
- return dim;
- }
- return null;
- }
-
- public DimensionDesc findDimensionByTable(String lookupTableName) {
- lookupTableName = lookupTableName.toUpperCase();
- for (DimensionDesc dim : dimensions)
- if (dim.getTable() != null && dim.getTable().equals(lookupTableName))
- return dim;
- return null;
- }
-
- public DimensionDesc findDimensionByName(String dimName) {
- dimName = dimName.toUpperCase();
- for (DimensionDesc dim : dimensions) {
- if (dimName.equals(dim.getName()))
- return dim;
- }
- return null;
- }
-
- public TblColRef findPKByFK(TblColRef fk) {
- assert isFactTable(fk.getTable());
-
- TblColRef candidate = null;
-
- for (DimensionDesc dim : dimensions) {
- JoinDesc join = dim.getJoin();
- if (join == null)
- continue;
-
- int find = ArrayUtils.indexOf(join.getForeignKeyColumns(), fk);
- if (find >= 0) {
- candidate = join.getPrimaryKeyColumns()[find];
- if (join.getForeignKeyColumns().length == 1) { // is single
- // column join?
- break;
- }
- }
- }
- return candidate;
- }
-
- /**
- * Get all functions from each measure.
- *
- * @return
- */
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measures) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
- public List<TableDesc> listTables() {
- MetadataManager metaMgr = MetadataManager.getInstance(config);
- HashSet<String> tableNames = new HashSet<String>();
- List<TableDesc> result = new ArrayList<TableDesc>();
-
- tableNames.add(factTable.toUpperCase());
- for (DimensionDesc dim : dimensions) {
- String table = dim.getTable();
- if (table != null)
- tableNames.add(table.toUpperCase());
- }
-
- for (String tableName : tableNames) {
- result.add(metaMgr.getTableDesc(tableName));
- }
-
- return result;
- }
-
- public boolean isFactTable(String factTable) {
- return this.factTable.equalsIgnoreCase(factTable);
- }
-
- public boolean isDerived(TblColRef col) {
- return derivedToHostMap.containsKey(col);
- }
-
- public DeriveInfo getHostInfo(TblColRef derived) {
- return derivedToHostMap.get(derived);
- }
-
- public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
- Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
- for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
- Array<TblColRef> hostCols = entry.getKey();
- boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
- if (!hostOnRow)
- continue;
-
- List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
- for (DeriveInfo info : entry.getValue()) {
- if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
- // any
- // wanted
- // columns?
- wantedInfo.add(info);
- }
-
- if (wantedInfo.size() > 0)
- result.put(hostCols, wantedInfo);
- }
- return result;
- }
-
- public String getResourcePath() {
- return getCubeDescResourcePath(name);
- }
-
- public static String getCubeDescResourcePath(String descName) {
- return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
- }
-
- // ============================================================================
-
- public HBaseMappingDesc getHBaseMapping() {
- return hbaseMapping;
- }
-
- public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public String getFactTable() {
- return factTable;
- }
-
- public void setFactTable(String factTable) {
- this.factTable = factTable;
- }
-
- public String[] getNullStrings() {
- return nullStrings;
- }
-
- public String getFilterCondition() {
- return filterCondition;
- }
-
- public void setFilterCondition(String filterCondition) {
- this.filterCondition = filterCondition;
- }
-
- public CubePartitionDesc getCubePartitionDesc() {
- return cubePartitionDesc;
- }
-
- public void setCubePartitionDesc(CubePartitionDesc cubePartitionDesc) {
- this.cubePartitionDesc = cubePartitionDesc;
- }
-
- public List<DimensionDesc> getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(List<DimensionDesc> dimensions) {
- this.dimensions = dimensions;
- }
-
- public List<MeasureDesc> getMeasures() {
- return measures;
- }
-
- public void setMeasures(List<MeasureDesc> measures) {
- this.measures = measures;
- }
-
- public RowKeyDesc getRowkey() {
- return rowkey;
- }
-
- public void setRowkey(RowKeyDesc rowkey) {
- this.rowkey = rowkey;
- }
-
- public String getSignature() {
- return signature;
- }
-
- public void setSignature(String signature) {
- this.signature = signature;
- }
-
- public CubeCapacity getCapacity() {
- return capacity;
- }
-
- public void setCapacity(CubeCapacity capacity) {
- this.capacity = capacity;
- }
-
- public List<String> getNotifyList() {
- return notifyList;
- }
-
- public void setNotifyList(List<String> notifyList) {
- this.notifyList = notifyList;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- CubeDesc cubeDesc = (CubeDesc) o;
-
- if (!name.equals(cubeDesc.name))
- return false;
- if (!factTable.equals(cubeDesc.factTable))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = 0;
- result = 31 * result + name.hashCode();
- result = 31 * result + factTable.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "CubeDesc [name=" + name + ", factTable=" + factTable + ", cubePartitionDesc=" + cubePartitionDesc + ", dimensions=" + dimensions + ", measures=" + measures + ", rowkey=" + rowkey + ", hbaseMapping=" + hbaseMapping + "]";
- }
-
- public String calculateSignature() {
- MessageDigest md = null;
- try {
- md = MessageDigest.getInstance("MD5");
- StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.factTable).append("|").append(JsonUtil.writeValueAsString(this.cubePartitionDesc)).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
-
- byte[] signature = md.digest(sigString.toString().getBytes());
- return new String(Base64.encodeBase64(signature));
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("Failed to calculate signature");
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to calculate signature");
- }
- }
-
- public Map<String, TblColRef> buildColumnNameAbbreviation() {
- Map<String, TblColRef> r = new HashMap<String, TblColRef>();
- for (TblColRef col : listDimensionColumnsExcludingDerived()) {
- r.put(col.getName(), col);
- }
- return r;
- }
-
- public void init(KylinConfig config, Map<String, TableDesc> tables) {
- this.errors.clear();
- this.config = config;
-
- if (factTable != null)
- factTable = factTable.toUpperCase();
-
- for (DimensionDesc dim : dimensions) {
- dim.init(tables);
- }
-
- sortDimAndMeasure();
-
- initJoinColumns(tables);
- initDimensionColumns(tables);
- initMeasureColumns(tables);
-
- /** TODO: should do the initialization after conver
- rowkey.init(this);
- if (hbaseMapping != null) {
- hbaseMapping.init(this);
- }
- */
- initMeasureReferenceToColumnFamily();
-
- if (null != this.cubePartitionDesc) {
- this.cubePartitionDesc.init(columnMap);
- }
-
- // check all dimension columns are presented on rowkey
- List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
- if (rowkey.getRowKeyColumns().length != dimCols.size()) {
- addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). ");
- }
- }
-
- private void initDimensionColumns(Map<String, TableDesc> tables) {
- // fill back ColRefDesc
- for (DimensionDesc dim : dimensions) {
- TableDesc dimTable = tables.get(dim.getTable());
- JoinDesc join = dim.getJoin();
-
- ArrayList<TblColRef> dimColList = new ArrayList<TblColRef>();
- ArrayList<TblColRef> hostColList = new ArrayList<TblColRef>();
-
- // dimension column
- if (dim.getColumn() != null) {
- if ("{FK}".equals(dim.getColumn())) {
- for (TblColRef ref : join.getForeignKeyColumns()) {
- TblColRef inited = initDimensionColRef(ref);
- dimColList.add(inited);
- hostColList.add(inited);
- }
- } else {
- TblColRef ref = initDimensionColRef(dimTable, dim.getColumn());
- dimColList.add(ref);
- hostColList.add(ref);
- }
- }
- // hierarchy columns
- if (dim.getHierarchy() != null) {
- for (HierarchyDesc hier : dim.getHierarchy()) {
- TblColRef ref = initDimensionColRef(dimTable, hier.getColumn());
- hier.setColumnRef(ref);
- dimColList.add(ref);
- }
- if (hostColList.isEmpty()) { // the last hierarchy could serve
- // as host when col is
- // unspecified
- hostColList.add(dimColList.get(dimColList.size() - 1));
- }
- }
- TblColRef[] dimCols = (TblColRef[]) dimColList.toArray(new TblColRef[dimColList.size()]);
- dim.setColumnRefs(dimCols);
-
- // lookup derived columns
- TblColRef[] hostCols = (TblColRef[]) hostColList.toArray(new TblColRef[hostColList.size()]);
- String[] derived = dim.getDerived();
- if (derived != null) {
- String[][] split = splitDerivedColumnAndExtra(derived);
- String[] derivedNames = split[0];
- String[] derivedExtra = split[1];
- TblColRef[] derivedCols = new TblColRef[derivedNames.length];
- for (int i = 0; i < derivedNames.length; i++) {
- derivedCols[i] = initDimensionColRef(dimTable, derivedNames[i]);
- }
- initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
- }
-
- // FK derived column
- if (join != null) {
- TblColRef[] fk = join.getForeignKeyColumns();
- TblColRef[] pk = join.getPrimaryKeyColumns();
- for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, fk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
- for (int i = 0; i < pk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, pk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(fk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
- }
- }
- }
-
- private String[][] splitDerivedColumnAndExtra(String[] derived) {
- String[] cols = new String[derived.length];
- String[] extra = new String[derived.length];
- for (int i = 0; i < derived.length; i++) {
- String str = derived[i];
- int cut = str.indexOf(":");
- if (cut >= 0) {
- cols[i] = str.substring(0, cut);
- extra[i] = str.substring(cut + 1).trim();
- } else {
- cols[i] = str;
- extra[i] = "";
- }
- }
- return new String[][] { cols, extra };
- }
-
- private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) {
- initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null);
- }
-
- private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
- if (hostCols.length == 0 || derivedCols.length == 0)
- throw new IllegalStateException("host/derived columns must not be empty");
-
- Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
- List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
- if (infoList == null) {
- hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
- }
- infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
-
- for (int i = 0; i < derivedCols.length; i++) {
- TblColRef derivedCol = derivedCols[i];
- boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
- }
- }
-
- private TblColRef initDimensionColRef(TableDesc table, String colName) {
- ColumnDesc col = table.findColumnByName(colName);
- if (col == null)
- throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
-
- TblColRef ref = new TblColRef(col);
- return initDimensionColRef(ref);
- }
-
- private TblColRef initDimensionColRef(TblColRef ref) {
- TblColRef existing = findColumnRef(ref.getTable(), ref.getName());
- if (existing != null) {
- return existing;
- }
-
- allColumns.add(ref);
- dimensionColumns.add(ref);
-
- Map<String, TblColRef> cols = columnMap.get(ref.getTable());
- if (cols == null) {
- columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>());
- }
- cols.put(ref.getName(), ref);
- return ref;
- }
-
- private void initJoinColumns(Map<String, TableDesc> tables) {
- // join columns may or may not present in cube;
- // here we don't modify 'allColumns' and 'dimensionColumns';
- // initDimensionColumns() will do the update
- for (DimensionDesc dim : dimensions) {
- TableDesc dimTable = tables.get(dim.getTable());
-
- JoinDesc join = dim.getJoin();
- if (join == null)
- continue;
-
- // primary key
- String[] pks = join.getPrimaryKey();
- TblColRef[] pkCols = new TblColRef[pks.length];
- for (int i = 0; i < pks.length; i++) {
- ColumnDesc col = dimTable.findColumnByName(pks[i]);
- if (col == null) {
- addError("Can't find column " + pks[i] + " in table " + dimTable.getName());
- }
- TblColRef colRef = new TblColRef(col);
- pks[i] = colRef.getName();
- pkCols[i] = colRef;
- }
- join.setPrimaryKeyColumns(pkCols);
- // foreign key
- TableDesc factTable = tables.get(this.factTable);
- if (factTable == null) {
- addError("Fact table does not exist:" + this.factTable);
- }
- String[] fks = join.getForeignKey();
- TblColRef[] fkCols = new TblColRef[fks.length];
- for (int i = 0; i < fks.length; i++) {
- ColumnDesc col = factTable.findColumnByName(fks[i]);
- if (col == null) {
- addError("Can't find column " + fks[i] + " in table " + this.factTable);
- }
- TblColRef colRef = new TblColRef(col);
- fks[i] = colRef.getName();
- fkCols[i] = colRef;
- }
- join.setForeignKeyColumns(fkCols);
- // Validate join in dimension
- if (pkCols.length != fkCols.length) {
- addError("Primary keys(" + dim.getTable() + ")" + Arrays.toString(pks) + " are not consistent with Foreign keys(" + this.factTable + ") " + Arrays.toString(fks));
- }
- for (int i = 0; i < fkCols.length; i++) {
- if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) {
- addError("Primary key " + dim.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.factTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
- }
- }
-
- }
- }
-
- private void initMeasureColumns(Map<String, TableDesc> tables) {
- if (measures == null || measures.isEmpty()) {
- return;
- }
-
- TableDesc factTable = tables.get(getFactTable());
- for (MeasureDesc m : measures) {
- m.setName(m.getName().toUpperCase());
-
- if (m.getDependentMeasureRef() != null) {
- m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
- }
-
- FunctionDesc f = m.getFunction();
- f.setExpression(f.getExpression().toUpperCase());
- f.initReturnDataType();
-
- ParameterDesc p = f.getParameter();
-
- if (p.isColumnType()) {
- ArrayList<TblColRef> colRefs = Lists.newArrayList();
- for (String cName : p.getValue().split("\\s*,\\s*")) {
- ColumnDesc sourceColumn = factTable.findColumnByName(cName);
- TblColRef colRef = new TblColRef(sourceColumn);
- colRefs.add(colRef);
- allColumns.add(colRef);
- }
- if (colRefs.isEmpty() == false)
- p.setColRefs(colRefs);
- }
- }
- }
-
- private void initMeasureReferenceToColumnFamily() {
- if (measures == null || measures.size() == 0)
- return;
-
- Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
- for (MeasureDesc m : measures)
- measureCache.put(m.getName(), m);
-
- for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
- for (HBaseColumnDesc c : cf.getColumns()) {
- MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
- for (int i = 0; i < c.getMeasureRefs().length; i++) {
- measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
- }
- c.setMeasures(measureDescs);
- c.setColumnFamilyName(cf.getName());
- }
- }
- }
-
- private void sortDimAndMeasure() {
- sortDimensionsByID();
- sortMeasuresByID();
- for (DimensionDesc dim : dimensions) {
- sortHierarchiesByLevel(dim.getHierarchy());
- }
- }
-
- private void sortDimensionsByID() {
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc d1, DimensionDesc d2) {
- Integer id1 = d1.getId();
- Integer id2 = d2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortMeasuresByID() {
- if (measures == null) {
- measures = Lists.newArrayList();
- }
-
- Collections.sort(measures, new Comparator<MeasureDesc>() {
- @Override
- public int compare(MeasureDesc m1, MeasureDesc m2) {
- Integer id1 = m1.getId();
- Integer id2 = m2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
- if (hierarchies != null) {
- Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
- @Override
- public int compare(HierarchyDesc h1, HierarchyDesc h2) {
- Integer level1 = Integer.parseInt(h1.getLevel());
- Integer level2 = Integer.parseInt(h2.getLevel());
- return level1.compareTo(level2);
- }
- });
- }
- }
-
-
- /**
- * Add error info and thrown exception out
- *
- * @param message
- */
- public void addError(String message) {
- addError(message, false);
- }
-
- /**
- * @param message
- * error message
- * @param silent
- * if throw exception
- */
- public void addError(String message, boolean silent) {
- if (!silent) {
- throw new IllegalStateException(message);
- } else {
- this.errors.add(message);
- }
- }
-
- public List<String> getError() {
- return this.errors;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeInstance.java
deleted file mode 100644
index 9778ab7..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeInstance.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeInstance extends RootPersistentEntity {
-
- public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
- CubeInstance cubeInstance = new CubeInstance();
-
- cubeInstance.setConfig(cubeDesc.getConfig());
- cubeInstance.setName(cubeName);
- cubeInstance.setDescName(cubeDesc.getName());
- cubeInstance.setCreateTime(formatTime(System.currentTimeMillis()));
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
- cubeInstance.setStatus(CubeStatusEnum.DISABLED);
- cubeInstance.updateRandomUuid();
-
- return cubeInstance;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private CubeStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<CubeSegment> segments = new ArrayList<CubeSegment>();
-
- @JsonProperty("create_time")
- private String createTime;
-
- public List<CubeSegment> getBuildingSegments() {
- List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
- if (null != segments) {
- for (CubeSegment segment : segments) {
- if (CubeSegmentStatusEnum.NEW == segment.getStatus() || CubeSegmentStatusEnum.READY_PENDING == segment.getStatus()) {
- buildingSegments.add(segment);
- }
- }
- }
-
- return buildingSegments;
- }
-
- public long getAllocatedEndDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(segments.size() - 1).getDateRangeEnd();
- }
-
- public long getAllocatedStartDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(0).getDateRangeStart();
- }
-
- public List<CubeSegment> getMergingSegments() {
- return this.getMergingSegments(null);
- }
-
- public List<CubeSegment> getMergingSegments(CubeSegment cubeSegment) {
- CubeSegment buildingSegment;
- if (cubeSegment == null) {
- List<CubeSegment> buildingSegments = getBuildingSegments();
- if (buildingSegments.size() == 0) {
- return Collections.emptyList();
- }
- buildingSegment = buildingSegments.get(0);
- } else {
- buildingSegment = cubeSegment;
- }
-
- List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>();
- if (null != this.segments) {
- for (CubeSegment segment : this.segments) {
- if (segment.getStatus() == CubeSegmentStatusEnum.READY) {
- if (buildingSegment.getDateRangeStart() <= segment.getDateRangeStart() && buildingSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) {
- mergingSegments.add(segment);
- }
- }
- }
- }
- return mergingSegments;
-
- }
-
- public List<CubeSegment> getRebuildingSegments() {
- List<CubeSegment> buildingSegments = getBuildingSegments();
- if (buildingSegments.size() == 0) {
- return Collections.emptyList();
- } else {
- List<CubeSegment> rebuildingSegments = new ArrayList<CubeSegment>();
- if (null != this.segments) {
- long startDate = buildingSegments.get(0).getDateRangeStart();
- long endDate = buildingSegments.get(buildingSegments.size() - 1).getDateRangeEnd();
- for (CubeSegment segment : this.segments) {
- if (segment.getStatus() == CubeSegmentStatusEnum.READY) {
- if (startDate >= segment.getDateRangeStart() && startDate < segment.getDateRangeEnd() && segment.getDateRangeEnd() < endDate) {
- rebuildingSegments.add(segment);
- continue;
- }
- if (startDate <= segment.getDateRangeStart() && endDate >= segment.getDateRangeEnd()) {
- rebuildingSegments.add(segment);
- continue;
- }
- }
- }
- }
-
- return rebuildingSegments;
- }
- }
-
- // public CubeDesc getDescriptor() {
- // return CubeDescManager.getInstance(config).getCubeDesc(descName);
- // }
-
- public boolean isReady() {
- return getStatus() == CubeStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return "Cube [name=" + name + "]";
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("source_records_count")
- public long getSourceRecordCount() {
- long sizeRecordCount = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getSourceRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("source_records_size")
- public long getSourceRecordSize() {
- long sizeRecordSize = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getSourceRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName.toUpperCase();
- }
-
- public String getOriginDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public void setCost(int cost) {
- this.cost = cost;
- }
-
- public CubeStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(CubeStatusEnum status) {
- this.status = status;
- }
-
- public CubeSegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public CubeSegment getLatestReadySegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getStatus() != CubeSegmentStatusEnum.READY)
- continue;
- if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
- latest = seg;
- }
- }
- return latest;
- }
-
- public List<CubeSegment> getSegments() {
- return segments;
- }
-
- public List<CubeSegment> getSegments(CubeSegmentStatusEnum status) {
- List<CubeSegment> result = new ArrayList<CubeSegment>();
-
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public List<CubeSegment> getSegment(CubeSegmentStatusEnum status) {
- List<CubeSegment> result = Lists.newArrayList();
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
- return result;
- }
-
- public CubeSegment getSegment(String name, CubeSegmentStatusEnum status) {
- for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<CubeSegment> segments) {
- this.segments = segments;
- }
-
- public CubeSegment getSegmentById(String segmentId) {
- for (CubeSegment segment : segments) {
- if (Objects.equal(segment.getUuid(), segmentId)) {
- return segment;
- }
- }
- return null;
- }
-
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
-
- public long[] getDateRange() {
- List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
- if (readySegments.isEmpty()) {
- return new long[] { 0L, 0L };
- }
- long start = Long.MAX_VALUE;
- long end = Long.MIN_VALUE;
- for (CubeSegment segment : readySegments) {
- if (segment.getDateRangeStart() < start) {
- start = segment.getDateRangeStart();
- }
- if (segment.getDateRangeEnd() > end) {
- end = segment.getDateRangeEnd();
- }
- }
- return new long[] { start, end };
- }
-
- // private boolean appendOnHll() {
- // CubePartitionDesc cubePartitionDesc = getDescriptor().getCubePartitionDesc();
- // if (cubePartitionDesc == null) {
- // return false;
- // }
- // if (cubePartitionDesc.getPartitionDateColumn() == null) {
- // return false;
- // }
- // if (cubePartitionDesc.getCubePartitionType() != CubePartitionDesc.CubePartitionType.APPEND) {
- // return false;
- // }
- // return getDescriptor().hasHolisticCountDistinctMeasures();
- // }
-
- // public boolean appendBuildOnHllMeasure(long startDate, long endDate) {
- // if (!appendOnHll()) {
- // return false;
- // }
- // List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
- // if (readySegments.isEmpty()) {
- // return false;
- // }
- // for (CubeSegment readySegment: readySegments) {
- // if (readySegment.getDateRangeStart() == startDate && readySegment.getDateRangeEnd() == endDate) {
- // //refresh build
- // return false;
- // }
- // }
- // return true;
- // }
-
- // public boolean needMergeImmediatelyAfterBuild(CubeSegment segment) {
- // if (!appendOnHll()) {
- // return false;
- // }
- // List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
- // if (readySegments.isEmpty()) {
- // return false;
- // }
- // for (CubeSegment readySegment: readySegments) {
- // if (readySegment.getDateRangeEnd() > segment.getDateRangeStart()) {
- // //has overlap and not refresh
- // return true;
- // }
- // }
- // return false;
- // }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubePartitionDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubePartitionDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubePartitionDesc.java
deleted file mode 100644
index 83e1758..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubePartitionDesc.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author xduo
- *
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubePartitionDesc {
-
- public static enum CubePartitionType {
- APPEND, UPDATE_INSERT
- }
-
- @JsonProperty("partition_date_column")
- private String partitionDateColumn;
- @JsonProperty("partition_date_start")
- private long partitionDateStart = 0L;
- @JsonProperty("cube_partition_type")
- private CubePartitionType cubePartitionType = CubePartitionType.APPEND;
-
- private TblColRef partitionDateColumnRef;
-
- public void init(Map<String, Map<String, TblColRef>> columnMap) {
- if (null != partitionDateColumn) {
- String[] columns = partitionDateColumn.split("\\.");
-
- if (null != columns && columns.length == 2) {
- Map<String, TblColRef> cols = columnMap.get(columns[0].toUpperCase());
- if (cols != null)
- partitionDateColumnRef = cols.get(columns[1].toUpperCase());
-
- }
- }
- }
-
- public boolean isPartitioned() {
- return partitionDateColumnRef != null;
- }
-
- public String getPartitionDateColumn() {
- return partitionDateColumn;
- }
-
- public void setPartitionDateColumn(String partitionDateColumn) {
- this.partitionDateColumn = partitionDateColumn;
- }
-
- public long getPartitionDateStart() {
- return partitionDateStart;
- }
-
- public void setPartitionDateStart(long partitionDateStart) {
- this.partitionDateStart = partitionDateStart;
- }
-
- public CubePartitionType getCubePartitionType() {
- return cubePartitionType;
- }
-
- public void setCubePartitionType(CubePartitionType cubePartitionType) {
- this.cubePartitionType = cubePartitionType;
- }
-
- public TblColRef getPartitionDateColumnRef() {
- return partitionDateColumnRef;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegment.java
deleted file mode 100644
index 83ef94c..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegment.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment> {
-
- @JsonBackReference
- private CubeInstance cubeInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private CubeSegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("source_records")
- private long sourceRecords;
- @JsonProperty("source_records_size")
- private long sourceRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
- @JsonProperty("create_time")
- private String createTime;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of cube schema and dictionary ID,
- // used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path
- @JsonProperty("snapshots")
- private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
-
- // public CubeDesc getCubeDesc() {
- // return getCubeInstance().getDescriptor();
- // }
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public CubeSegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(CubeSegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getSourceRecords() {
- return sourceRecords;
- }
-
- public void setSourceRecords(long sourceRecords) {
- this.sourceRecords = sourceRecords;
- }
-
- public long getSourceRecordsSize() {
- return sourceRecordsSize;
- }
-
- public void setSourceRecordsSize(long sourceRecordsSize) {
- this.sourceRecordsSize = sourceRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public CubeInstance getCubeInstance() {
- return cubeInstance;
- }
-
- public void setCubeInstance(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- }
-
- public String getStorageLocationIdentifier() {
-
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Map<String, String> getSnapshots() {
- if (snapshots == null)
- snapshots = new ConcurrentHashMap<String, String>();
- return snapshots;
- }
-
- public String getSnapshotResPath(String table) {
- return getSnapshots().get(table);
- }
-
- public void putSnapshotResPath(String table, String snapshotResPath) {
- getSnapshots().put(table, snapshotResPath);
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public Collection<String> getSnapshotPaths() {
- return getSnapshots().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- /**
- * @param storageLocationIdentifier
- * the storageLocationIdentifier to set
- */
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int compareTo(CubeSegment other) {
- if (this.dateRangeEnd < other.dateRangeEnd) {
- return -1;
- } else if (this.dateRangeEnd > other.dateRangeEnd) {
- return 1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((status == null) ? 0 : status.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CubeSegment other = (CubeSegment) obj;
- if (cubeInstance == null) {
- if (other.cubeInstance != null)
- return false;
- } else if (!cubeInstance.equals(other.cubeInstance))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (status != other.status)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time:", createTime).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentStatusEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentStatusEnum.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentStatusEnum.java
deleted file mode 100644
index 62346f8..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentStatusEnum.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-/**
- * @author xduo
- *
- */
-public enum CubeSegmentStatusEnum {
- NEW, READY, READY_PENDING
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentTypeEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentTypeEnum.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentTypeEnum.java
deleted file mode 100644
index 3b7350d..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeSegmentTypeEnum.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-/**
- * @author ysong1
- *
- */
-public enum CubeSegmentTypeEnum {
- TRANSIENT, PERMANENT
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeStatusEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeStatusEnum.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeStatusEnum.java
deleted file mode 100644
index 58bf92b..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeStatusEnum.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-public enum CubeStatusEnum {
-
- DISABLED, BUILDING, READY, DESCBROKEN
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/DimensionDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/DimensionDesc.java
deleted file mode 100644
index ace1e3e..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/DimensionDesc.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.util.Arrays;
-import java.util.Map;
-
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc {
-
- @JsonProperty("id")
- private int id;
- @JsonProperty("name")
- private String name;
- @JsonProperty("join")
- private JoinDesc join;
- @JsonProperty("hierarchy")
- private HierarchyDesc[] hierarchy;
- @JsonProperty("table")
- private String table;
- @JsonProperty("column")
- private String column;
- @JsonProperty("datatype")
- private String datatype;
- @JsonProperty("derived")
- private String[] derived;
-
- // computed
- private TblColRef[] columnRefs;
- private TblColRef[] derivedColRefs;
-
- public boolean isHierarchyColumn(TblColRef col) {
- if (hierarchy == null)
- return false;
-
- for (HierarchyDesc hier : hierarchy) {
- if (hier.getColumnRef().equals(col))
- return true;
- }
- return false;
- }
-
- public String getDatatype() {
- return datatype;
- }
-
- public void setDatatype(String datatype) {
- this.datatype = datatype;
- }
-
- public String getTable() {
- return table.toUpperCase();
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
- public JoinDesc getJoin() {
- return join;
- }
-
- public void setJoin(JoinDesc join) {
- this.join = join;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public TblColRef[] getColumnRefs() {
- return this.columnRefs;
- }
-
- public void setColumnRefs(TblColRef[] colRefs) {
- this.columnRefs = colRefs;
- }
-
- public String getColumn() {
- return this.column;
- }
-
- public void setColumn(String column) {
- this.column = column;
- if (this.column != null)
- this.column = this.column.toUpperCase();
- }
-
- public HierarchyDesc[] getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(HierarchyDesc[] hierarchy) {
- this.hierarchy = hierarchy;
- }
-
- public String[] getDerived() {
- return derived;
- }
-
- public void setDerived(String[] derived) {
- this.derived = derived;
- }
-
- public TblColRef[] getDerivedColRefs() {
- return derivedColRefs;
- }
-
- public void setDerivedColRefs(TblColRef[] derivedColRefs) {
- this.derivedColRefs = derivedColRefs;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- DimensionDesc that = (DimensionDesc) o;
-
- if (id != that.id)
- return false;
- if (!name.equals(that.name))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = id;
- result = 31 * result + name.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + column + ", datatype=" + datatype + ", derived=" + Arrays.toString(derived) + "]";
- }
-
- public void init(Map<String, TableDesc> tables) {
- if (name != null)
- name = name.toUpperCase();
- if (table != null)
- table = table.toUpperCase();
- if (column != null)
- column = column.toUpperCase();
-
- TableDesc tableDesc = tables.get(table);
- if (tableDesc == null)
- throw new IllegalStateException("Can't find table " + table + " on dimension " + name);
-
- if (hierarchy != null && hierarchy.length == 0)
- hierarchy = null;
- if (derived != null && derived.length == 0)
- derived = null;
-
- if (join != null) {
- StringUtil.toUpperCaseArray(join.getForeignKey(), join.getForeignKey());
- StringUtil.toUpperCaseArray(join.getPrimaryKey(), join.getPrimaryKey());
- }
-
- if (hierarchy != null) {
- for (HierarchyDesc h : hierarchy)
- h.setColumn(h.getColumn().toUpperCase());
- }
-
- if (derived != null) {
- StringUtil.toUpperCaseArray(derived, derived);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/ProjectInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/ProjectInstance.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/ProjectInstance.java
deleted file mode 100644
index cb551c2..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/ProjectInstance.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.metadata.project.ProjectStatusEnum;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Project is a concept in Kylin similar to schema in DBMS
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class ProjectInstance extends RootPersistentEntity {
-
- public static final String DEFAULT_PROJECT_NAME = "DEFAULT";
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("cubes")
- private List<String> cubes;
-
- @JsonProperty("tables")
- private Set<String> tables;
-
- @JsonProperty("owner")
- private String owner;
-
- @JsonProperty("status")
- private ProjectStatusEnum status;
-
- @JsonProperty("create_time")
- private String createTime;
-
- @JsonProperty("last_update_time")
- private String lastUpdateTime;
-
- @JsonProperty("description")
- private String description;
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String projectName) {
- return ResourceStore.PROJECT_RESOURCE_ROOT + "/" + projectName + ".json";
- }
-
- public static String getNormalizedProjectName(String project) {
- if (project == null)
- throw new IllegalStateException("Trying to normalized a project name which is null");
-
- return project.toUpperCase();
- }
-
- // ============================================================================
-
- public static ProjectInstance create(String name, String owner, String description, List<String> cubes) {
- ProjectInstance projectInstance = new ProjectInstance();
-
- projectInstance.updateRandomUuid();
- projectInstance.setName(name);
- projectInstance.setOwner(owner);
- projectInstance.setDescription(description);
- projectInstance.setStatus(ProjectStatusEnum.ENABLED);
- projectInstance.setCreateTime(formatTime(System.currentTimeMillis()));
- if (cubes != null)
- projectInstance.setCubes(cubes);
- else
- projectInstance.setCubes(new ArrayList<String>());
-
- return projectInstance;
- }
-
- public ProjectInstance() {
-
- }
-
- public ProjectInstance(String name, List<String> cubes, String owner) {
- this.name = name;
- this.cubes = cubes;
- this.owner = owner;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public ProjectStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(ProjectStatusEnum status) {
- this.status = status;
- }
-
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
-
- public String getName() {
- return this.name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public boolean containsCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- return cubes.contains(cubeName);
- }
-
- public void removeCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- cubes.remove(cubeName);
- }
-
- public int getCubesCount() {
- return cubes.size();
- }
-
- public void addCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- this.cubes.add(cubeName);
- }
-
- public List<String> getCubes() {
- return cubes;
- }
-
- public void setCubes(List<String> cubes) {
- this.cubes = cubes;
- }
-
- public void setTables(Set<String> tables) {
- this.tables = tables;
- }
-
- public boolean containsTable(String tableName) {
- tableName = tableName.toUpperCase();
- return tables.contains(tableName);
- }
-
- public void removeTable(String tableName) {
- tableName = tableName.toUpperCase();
- tables.remove(tableName);
- }
-
- public int getTablesCount() {
- return this.getTables().size();
- }
-
- public void addTable(String tableName) {
- tableName = tableName.toUpperCase();
- this.getTables().add(tableName);
- }
-
- //will return new Set for null
- public Set<String> getTables() {
- tables = tables == null ? new TreeSet<String>() : tables;
- return tables;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getLastUpdateTime() {
- return lastUpdateTime;
- }
-
- public void setLastUpdateTime(String lastUpdateTime) {
- this.lastUpdateTime = lastUpdateTime;
- }
-
- public void recordUpdateTime(long timeMillis) {
- this.lastUpdateTime = formatTime(timeMillis);
- }
-
- public void init() {
- if (name == null)
- name = ProjectInstance.DEFAULT_PROJECT_NAME;
-
- if (cubes == null) {
- cubes = new ArrayList<String>();
- }
-
- for (int i = 0; i < cubes.size(); ++i) {
- if (cubes.get(i) != null)
- cubes.set(i, cubes.get(i).toUpperCase());
- }
- }
-
- @Override
- public String toString() {
- return "ProjectDesc [name=" + name + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
deleted file mode 100644
index 77b1770..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/CubeMetadataValidator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation;
-
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.apache.kylin.cube.model.validation.rule.FunctionRule;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
-
-/**
- * For cube metadata validator
- *
- * @author jianliu
- *
- */
-public class CubeMetadataValidator {
- @SuppressWarnings("unchecked")
- private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupSizeRule(), new MandatoryColumnRule(), new RowKeyAttrRule() };
-
- public ValidateContext validate(CubeDesc cube) {
- return validate(cube, false);
- }
-
- /**
- * @param cubeDesc
- * @param inject
- * inject error into cube desc
- * @return
- */
- public ValidateContext validate(CubeDesc cube, boolean inject) {
- ValidateContext context = new ValidateContext();
- for (int i = 0; i < rules.length; i++) {
- IValidatorRule<CubeDesc> rule = rules[i];
- rule.validate(cube, context);
- }
- if (inject) {
- injectResult(cube, context);
- }
- return context;
- }
-
- /**
- *
- * Inject errors info into cubeDesc
- *
- * @param cubeDesc
- * @param context
- */
- public void injectResult(CubeDesc cubeDesc, ValidateContext context) {
- ValidateContext.Result[] results = context.getResults();
- for (int i = 0; i < results.length; i++) {
- ValidateContext.Result result = results[i];
- cubeDesc.addError(result.getLevel() + " : " + result.getMessage(), true);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java
deleted file mode 100644
index 0368528..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/IValidatorRule.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation;
-
-import org.apache.kylin.cube.model.validation.rule.IKylinValidationConstants;
-
-/**
- * @author jianliu
- *
- */
-public interface IValidatorRule<T> extends IKylinValidationConstants {
-
- public void validate(T element, ValidateContext context);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java
deleted file mode 100644
index 9dcefc5..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/ResultLevel.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation;
-
-/**
- * Validation result level
- *
- * @author jianliu
- *
- */
-public enum ResultLevel {
- ERROR("ERROR"), WARN("WARN");
- private String level;
-
- ResultLevel(String level) {
- this.level = level;
- }
-
- public String toString() {
- return level;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/SourceTableMetadataValidator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/SourceTableMetadataValidator.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/SourceTableMetadataValidator.java
deleted file mode 100644
index 14be1c4..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/SourceTableMetadataValidator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation;
-
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * Validate Table metadata from source.
- * <p/>
- * User: lukhan Date: 12/2/13 Time: 10:45 AM To change this template use File |
- * Settings | File Templates.
- */
-public class SourceTableMetadataValidator {
-
- public static boolean validate(TableDesc table) {
- // table.get
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
deleted file mode 100644
index 8ac5360..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/ValidateContext.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation;
-
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Context. Supply all dependent objects for validator
- *
- * @author jianliu
- *
- */
-public class ValidateContext {
- private List<Result> results = new ArrayList<ValidateContext.Result>();
-
- public void addResult(ResultLevel level, String message) {
- results.add(new Result(level, message));
- }
-
- public void addResult(Result result) {
- results.add(result);
- }
-
- public class Result {
- private ResultLevel level;
- private String message;
-
- /**
- * @param level
- * @param message
- */
- public Result(ResultLevel level, String message) {
- this.level = level;
- this.message = message;
- }
-
- /**
- * @return the level
- */
- public ResultLevel getLevel() {
- return level;
- }
-
- /**
- * @return the message
- */
- public String getMessage() {
- return message;
- }
- }
-
- /**
- * Get validation result
- *
- * @return
- */
- public Result[] getResults() {
- Result[] rs = new Result[0];
- rs = results.toArray(rs);
- return rs;
- }
-
- /**
- *
- */
- public void print(PrintStream out) {
- if (results.isEmpty()) {
- out.print("The element is perfect.");
- }
- Iterator<Result> it = results.iterator();
- while (it.hasNext()) {
- Result result = it.next();
- out.println(result.level + " : " + result.message);
- }
- }
-
- /**
- * @return if there is not validation errors
- */
- public boolean ifPass() {
- return results.isEmpty();
- }
-
-}
[36/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index 23f24e0..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- *
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- innerValidateMaxSize(cube, context);
- }
-
- /**
- * @param cube
- * @param context
- */
- private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
- int maxSize = getMaxAgrGroupSize();
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- if (group.length >= maxSize) {
- context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
- }
- }
- }
-
- protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
deleted file mode 100644
index d7d9f13..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * Validate function parameter. Ticket:
- * https://github.scm.corp.ebay.com/Kylin/Kylin/issues/268
- * <p/>
- * if type is column, check values are valid fact table columns if type is
- * constant, the value only can be numberic
- * <p/>
- * the return type only can be int/bigint/long/double/decimal
- *
- * @author jianliu
- */
-public class FunctionRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- List<MeasureDesc> measures = cube.getMeasures();
-
- List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>();
-
- Iterator<MeasureDesc> it = measures.iterator();
- while (it.hasNext()) {
- MeasureDesc measure = it.next();
- FunctionDesc func = measure.getFunction();
- ParameterDesc parameter = func.getParameter();
- if (parameter == null) {
- context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
- return;
- }
-
- String type = func.getParameter().getType();
- String value = func.getParameter().getValue();
- if (StringUtils.isEmpty(type)) {
- context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
- return;
- }
- if (StringUtils.isEmpty(value)) {
- context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
- return;
- }
- if (StringUtils.isEmpty(func.getReturnType())) {
- context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
- return;
- }
-
- if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
- validateColumnParameter(context, cube, value);
- } else if (StringUtils.equals(FunctionDesc.PARAMTER_TYPE_CONSTANT, type)) {
- validateCostantParameter(context, cube, value);
- }
- try {
- func.getMeasureType().validate(func);
- } catch (IllegalArgumentException ex) {
- context.addResult(ResultLevel.ERROR, ex.getMessage());
- }
-
- if (func.isCount())
- countFuncs.add(func);
- }
-
- if (countFuncs.size() != 1) {
- context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs);
- }
- }
-
- /**
- * @param context
- * @param cube
- * @param value
- */
- private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) {
- try {
- Integer.parseInt(value);
- } catch (Exception e) {
- context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value);
- }
- }
-
- /**
- * @param context
- * @param cube
- * @param value
- */
- private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
- String factTable = cube.getFactTable();
- if (StringUtils.isEmpty(factTable)) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
- return;
- }
- TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
- if (table == null) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
- return;
- }
- // Prepare column set
- Set<String> set = new HashSet<String>();
- ColumnDesc[] cdesc = table.getColumns();
- for (int i = 0; i < cdesc.length; i++) {
- ColumnDesc columnDesc = cdesc[i];
- set.add(columnDesc.getName());
- }
-
- String[] items = value.split(",");
- for (int i = 0; i < items.length; i++) {
- String item = items[i].trim();
- if (StringUtils.isEmpty(item)) {
- continue;
- }
- if (!set.contains(item)) {
- context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
deleted file mode 100644
index 95e92ad..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.metadata.MetadataConstants;
-
-/**
- * @author jianliu
- *
- */
-public interface IKylinValidationConstants extends MetadataConstants {
-
- public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
- public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
- public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index b0e6847..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- *
- * @author jianliu
- *
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- Set<String> mands = new HashSet<String>();
- RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
- if (cols == null || cols.length == 0) {
- return;
- }
- for (int i = 0; i < cols.length; i++) {
- RowKeyColDesc rowKeyColDesc = cols[i];
- if (rowKeyColDesc.isMandatory()) {
- mands.add(rowKeyColDesc.getColumn());
- }
- }
- if (mands.isEmpty()) {
- return;
- }
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- for (int j = 0; j < group.length; j++) {
- String col = group[j];
- if (mands.contains(col)) {
- context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
- }
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java b/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
deleted file mode 100644
index 9b8575f..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/validation/rule/RowKeyAttrRule.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-
-/**
- * Validate that only one of "length" and "dictionary" appears on rowkey_column
- *
- * @author jianliu
- *
- */
-public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- RowKeyDesc row = cube.getRowkey();
- if (row == null) {
- context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
- return;
- }
-
- RowKeyColDesc[] rcd = row.getRowKeyColumns();
- if (rcd == null) {
- context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
- return;
- }
- if (rcd.length == 0) {
- context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
- return;
- }
-
- for (int i = 0; i < rcd.length; i++) {
- RowKeyColDesc rd = rcd[i];
- if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
- }
- if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java b/cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index 4a5278e..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupSizeRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 3;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[2].getMessage()
- // .startsWith("Hierachy column"));
- }
-
- @Test
- public void testAggColumnSize() {
- AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.kylin.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 20;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Aggregation group"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Hierachy column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
deleted file mode 100644
index c0b6210..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- */
-public class CubeDescTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testSerialize() throws Exception {
- CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
- String str = JsonUtil.writeValueAsIndentString(desc);
- System.out.println(str);
- @SuppressWarnings("unused")
- CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class);
- }
-
- @Test
- public void testGetCubeDesc() throws Exception {
- CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
- Assert.assertNotNull(cubeDesc);
- }
-
- @Test
- public void testSerializeMap() throws Exception {
- Map<String, String> map = Maps.newHashMap();
-
- map.put("key1", "value1");
- map.put("key2", "value2");
-
- String mapStr = JsonUtil.writeValueAsString(map);
-
- System.out.println(mapStr);
-
- Map<?, ?> map2 = JsonUtil.readValue(mapStr, HashMap.class);
-
- Assert.assertEquals(map, map2);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
deleted file mode 100644
index 9796fb2..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
-
- private CubeManager cubeManager;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- CubeManager.clearCache();
- ProjectManager.clearCache();
- cubeManager = CubeManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testReloadCache() throws Exception {
- ResourceStore store = getStore();
-
- // clean legacy in case last run failed
- store.deleteResource("/cube/a_whole_new_cube.json");
- CubeDescManager cubeDescMgr = getCubeDescManager();
- CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- cubeManager.createCube("a_whole_new_cube", "default", desc, null);
-
- CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
- assertEquals(0, createdCube.getSegments().size());
- assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
- createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
-
- cubeManager.updateCube(createdCube);
- assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
- }
-
- public CubeDescManager getCubeDescManager() {
- return CubeDescManager.getInstance(getTestConfig());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
deleted file mode 100644
index 63ee2cc..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class CubeManagerTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testBasics() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
- CubeDesc desc = cube.getDescriptor();
- System.out.println(JsonUtil.writeValueAsIndentString(desc));
-
- String signature = desc.calculateSignature();
- desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column");
- assertTrue(!signature.equals(desc.calculateSignature()));
- }
-
- @Test
- public void testCreateAndDrop() throws Exception {
-
- KylinConfig config = getTestConfig();
- CubeManager cubeMgr = CubeManager.getInstance(config);
- ProjectManager prjMgr = ProjectManager.getInstance(config);
- ResourceStore store = getStore();
-
- // clean legacy in case last run failed
- store.deleteResource("/cube/a_whole_new_cube.json");
-
- CubeDescManager cubeDescMgr = getCubeDescManager();
- CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
- assertTrue(createdCube == cubeMgr.getCube("a_whole_new_cube"));
-
- assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));
-
- CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", true);
- assertTrue(createdCube == droppedCube);
-
- assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));
-
- assertNull(CubeManager.getInstance(getTestConfig()).getCube("a_whole_new_cube"));
- }
-
- public CubeDescManager getCubeDescManager() {
- return CubeDescManager.getInstance(getTestConfig());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
deleted file mode 100644
index 99e5003..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/CubeSegmentsTest.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CubeSegmentsTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test(expected = IllegalStateException.class)
- public void testAppendNonPartitioned() throws IOException {
- CubeManager mgr = mgr();
- CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_empty");
-
- // first append, creates a new & single segment
- CubeSegment seg = mgr.appendSegments(cube, 0);
- assertEquals(0, seg.getDateRangeStart());
- assertEquals(Long.MAX_VALUE, seg.getDateRangeEnd());
- assertEquals(1, cube.getSegments().size());
-
- // second append, throw IllegalStateException because the first segment is not built
- CubeSegment seg2 = mgr.appendSegments(cube, 0);
- discard(seg2);
- }
-
- @Test(expected = IllegalStateException.class)
- public void testAppendNonPartitioned2() throws IOException {
- CubeManager mgr = mgr();
- CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready");
-
- // assert one ready segment
- assertEquals(1, cube.getSegments().size());
- CubeSegment seg = cube.getSegment(SegmentStatusEnum.READY).get(0);
- assertEquals(SegmentStatusEnum.READY, seg.getStatus());
-
- // append again, for non-partitioned cube, it becomes a full refresh
- CubeSegment seg2 = mgr.appendSegments(cube, 0);
- assertEquals(0, seg2.getDateRangeStart());
- assertEquals(Long.MAX_VALUE, seg2.getDateRangeEnd());
- assertEquals(2, cube.getSegments().size());
-
- // non-partitioned cannot merge, throw exception
- mgr.mergeSegments(cube, 0, Long.MAX_VALUE, false);
- }
-
- @Test
- public void testPartitioned() throws IOException {
- CubeManager mgr = mgr();
- CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_left_join_empty");
-
- // no segment at first
- assertEquals(0, cube.getSegments().size());
-
- // append first
- CubeSegment seg1 = mgr.appendSegments(cube, 1000);
- seg1.setStatus(SegmentStatusEnum.READY);
-
- // append second
- CubeSegment seg2 = mgr.appendSegments(cube, 2000);
-
- assertEquals(2, cube.getSegments().size());
- assertEquals(1000, seg2.getDateRangeStart());
- assertEquals(2000, seg2.getDateRangeEnd());
- assertEquals(SegmentStatusEnum.NEW, seg2.getStatus());
- seg2.setStatus(SegmentStatusEnum.READY);
-
- // merge first and second
- CubeSegment merge = mgr.mergeSegments(cube, 0, 2000, true);
-
- assertEquals(3, cube.getSegments().size());
- assertEquals(0, merge.getDateRangeStart());
- assertEquals(2000, merge.getDateRangeEnd());
- assertEquals(SegmentStatusEnum.NEW, merge.getStatus());
-
- // segments are strictly ordered
- assertEquals(seg1, cube.getSegments().get(0));
- assertEquals(merge, cube.getSegments().get(1));
- assertEquals(seg2, cube.getSegments().get(2));
-
- // drop the merge
- cube.getSegments().remove(merge);
-
- // try merge at start/end at middle of segments
- CubeSegment merge2 = mgr.mergeSegments(cube, 500, 1500, true);
- assertEquals(3, cube.getSegments().size());
- assertEquals(0, merge2.getDateRangeStart());
- assertEquals(2000, merge2.getDateRangeEnd());
- }
-
- @Test
- public void testSpecialAppendAndMerge() throws IOException {
- CubeManager mgr = mgr();
- CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_left_join_empty");
-
- // no segment at first
- assertEquals(0, cube.getSegments().size());
-
- try {
- mgr.appendAndMergeSegments(cube, 1000);
- fail();
- } catch (IllegalStateException ex) {
- // expected, append and merge requires at least one segment
- }
-
- // append first
- CubeSegment seg1 = mgr.appendSegments(cube, 1000);
- seg1.setStatus(SegmentStatusEnum.READY);
-
- // append second and merge with first
- Pair<CubeSegment, CubeSegment> appendAndMergeSegments = mgr.appendAndMergeSegments(cube, 2000);
- CubeSegment seg2 = appendAndMergeSegments.getFirst();
- CubeSegment merge = appendAndMergeSegments.getSecond();
-
- assertEquals(3, cube.getSegments().size());
- assertEquals(0, seg1.getDateRangeStart());
- assertEquals(1000, seg1.getDateRangeEnd());
- assertEquals(1000, seg2.getDateRangeStart());
- assertEquals(2000, seg2.getDateRangeEnd());
- assertEquals(0, merge.getDateRangeStart());
- assertEquals(2000, merge.getDateRangeEnd());
-
- // segments are strictly ordered
- assertEquals(seg1, cube.getSegments().get(0));
- assertEquals(merge, cube.getSegments().get(1));
- assertEquals(seg2, cube.getSegments().get(2));
- }
-
- private void discard(Object o) {
- // throw away input parameter
- }
-
- private CubeManager mgr() {
- return CubeManager.getInstance(getTestConfig());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
deleted file mode 100644
index f093032..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashSet;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class DictionaryManagerTest extends LocalFileMetadataTestCase {
-
- DictionaryManager dictMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- dictMgr = DictionaryManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- @Ignore("hive not ready")
- public void basic() throws Exception {
- CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
- TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
-
- DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
- System.out.println(JsonUtil.writeValueAsIndentString(info1));
-
- DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null);
- System.out.println(JsonUtil.writeValueAsIndentString(info2));
-
- assertTrue(info1.getUuid() == info2.getUuid());
-
- assertTrue(info1 == dictMgr.getDictionaryInfo(info1.getResourcePath()));
- assertTrue(info2 == dictMgr.getDictionaryInfo(info2.getResourcePath()));
-
- assertTrue(info1.getDictionaryObject() == info2.getDictionaryObject());
-
- touchDictValues(info1);
- }
-
- @SuppressWarnings("unchecked")
- private void touchDictValues(DictionaryInfo info1) {
- Dictionary<String> dict = (Dictionary<String>) info1.getDictionaryObject();
-
- HashSet<String> set = new HashSet<String>();
- for (int i = 0, n = info1.getCardinality(); i < n; i++) {
- set.add(dict.getValueFromId(i));
- }
- assertEquals(info1.getCardinality(), set.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java b/cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
deleted file mode 100644
index a85522c..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.MandatoryColumnRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class MandatoryColumnRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
- rule.validate(cube, vContext);
- assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java b/cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
deleted file mode 100644
index f1445ba..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/RowKeyAttrRuleTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.RowKeyAttrRule;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author jianliu
- *
- */
-public class RowKeyAttrRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new RowKeyAttrRule();
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertTrue("Failed to validate rowkey", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java b/cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java
deleted file mode 100644
index 3954ee7..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/common/BytesSplitterTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.common;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.util.BytesSplitter;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class BytesSplitterTest {
-
- @Test
- public void test() {
- BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
- byte[] input = "2013-02-17Collectibles".getBytes();
- bytesSplitter.split(input, input.length, (byte) 127);
-
- assertEquals(2, bytesSplitter.getBufferSize());
- assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
- assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
- }
-
- @Test
- public void testNullValue() {
- BytesSplitter bytesSplitter = new BytesSplitter(10, 15);
- byte[] input = "2013-02-17Collectibles".getBytes();
- bytesSplitter.split(input, input.length, (byte) 127);
-
- assertEquals(3, bytesSplitter.getBufferSize());
- assertEquals("2013-02-17", new String(bytesSplitter.getSplitBuffers()[0].value, 0, bytesSplitter.getSplitBuffers()[0].length));
- assertEquals("", new String(bytesSplitter.getSplitBuffers()[1].value, 0, bytesSplitter.getSplitBuffers()[1].length));
- assertEquals("Collectibles", new String(bytesSplitter.getSplitBuffers()[2].value, 0, bytesSplitter.getSplitBuffers()[2].length));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
deleted file mode 100644
index 3615f5c..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.common;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeySplitterTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testWithSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
-
- RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
- // base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- rowKeySplitter.split(input, input.length);
-
- assertEquals(10, rowKeySplitter.getBufferSize());
- }
-
- @Test
- public void testWithoutSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
-
- RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
- // base cuboid rowkey
- byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
- rowKeySplitter.split(input, input.length);
-
- assertEquals(9, rowKeySplitter.getBufferSize());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java b/cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java
deleted file mode 100644
index 4c43a46..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/cuboid/CombinationTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-import static org.junit.Assert.assertTrue;
-
-import org.junit.Test;
-
-/**
- * @author xjiang
- *
- */
-public class CombinationTest {
-
- public int findSmallerSibling(long valueBits, long valueMask) {
- if ((valueBits | valueMask) != valueMask) {
- throw new IllegalArgumentException("Dismatch " + Long.toBinaryString(valueBits) + " from " + Long.toBinaryString(valueMask));
- }
-
- int n = Long.bitCount(valueMask);
- int k = Long.bitCount(valueBits);
- long[] bitMasks = new long[n];
- long leftBits = valueMask;
- for (int i = 0; i < n; i++) {
- long lowestBit = Long.lowestOneBit(leftBits);
- bitMasks[i] = lowestBit;
- leftBits &= ~lowestBit;
- }
- return combination(valueBits, bitMasks, 0, 0L, k);
- }
-
- private int combination(long valueBits, long[] bitMasks, int offset, long prefix, int k) {
- if (k == 0) {
- if (prefix < valueBits) {
- System.out.println(Long.toBinaryString(prefix));
- return 1;
- } else {
- return 0;
- }
- } else {
- int count = 0;
- for (int i = offset; i < bitMasks.length; i++) {
- long newPrefix = prefix | bitMasks[i];
- if (newPrefix < valueBits) {
- count += combination(valueBits, bitMasks, i + 1, newPrefix, k - 1);
- }
- }
- return count;
- }
- }
-
- private long calculateCombination(int n, int k) {
- if (n < k) {
- throw new IllegalArgumentException("N < K");
- }
- long res = 1;
- for (int i = n - k + 1; i <= n; i++) {
- res *= i;
- }
- for (int i = 1; i <= k; i++) {
- res /= i;
- }
- return res;
- }
-
- @Test
- public void testComb3() {
- long valueBits = 1 << 4 | 1 << 6 | 1 << 8;
- System.out.println("value = " + Long.toBinaryString(valueBits) + ", count = " + Long.bitCount(valueBits));
- long valueMask = (long) Math.pow(2, 10) - 1;
- System.out.println("mask = " + Long.toBinaryString(valueMask) + ", count = " + Long.bitCount(valueMask));
- System.out.println("************");
- int count = findSmallerSibling(valueBits, valueMask);
- System.out.println("smaller sibling count = " + count);
- int cnk = (int) calculateCombination(Long.bitCount(valueMask), Long.bitCount(valueBits));
- assertTrue(cnk > count);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
deleted file mode 100644
index 52e34ec..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- static long toLong(String bin) {
- return Long.parseLong(bin, 2);
- }
-
- static String toString(long l) {
- return Long.toBinaryString(l);
- }
-
- static String toString(Collection<Long> cuboids) {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for (Long l : cuboids) {
- if (buf.length() > 1)
- buf.append(",");
- buf.append(l).append("(").append(Long.toBinaryString(l)).append(")");
- }
- buf.append("]");
- return buf.toString();
- }
-
- private CubeDesc getTestKylinCubeWithoutSeller() {
- return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithSeller() {
- return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() {
- return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
- }
-
- @Test
- public void testFindSmallerSibling1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(255);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(133);
- assertEquals("[131]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(127);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(134);
- assertEquals("[131, 133]", sortToString(siblings));
-
- siblings = scheduler.findSmallerSibling(130);
- assertEquals("[129]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(5);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(135);
- assertEquals("[]", siblings.toString());
- }
-
- private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds) {
- for (long cuboidId : cuboidIds) {
- Collection<Long> spannings = scheduler.getSpanningCuboid(cuboidId);
- System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));
-
- for (long child : spannings) {
- assertTrue(Cuboid.isValid(cube, child));
- }
- }
- }
-
- @Test
- public void testFindSmallerSibling2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> siblings;
-
- siblings = scheduler.findSmallerSibling(511);
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110111111"));
- assertEquals("[383]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("101110111"));
- assertEquals("[319]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("111111000"));
- assertEquals("[]", siblings.toString());
-
- siblings = scheduler.findSmallerSibling(toLong("110000000"));
- assertEquals("[288, 320]", sortToString(siblings));
- }
-
- @Test
- public void testGetSpanningCuboid2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- // generate 8d
- System.out.println("Spanning for 8D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 511 });
- // generate 7d
- System.out.println("Spanning for 7D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 });
- // generate 6d
- System.out.println("Spanning for 6D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 });
- // generate 5d
- System.out.println("Spanning for 5D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 });
- // generate 4d
- System.out.println("Spanning for 4D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 });
- // generate 3d
- System.out.println("Spanning for 3D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 384, 288, 320, 263 });
- // generate 2d
- // generate 1d
- // generate 0d
- }
-
- @Test
- public void testGetSpanningCuboid1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- long quiz = toLong("01100111");
- testSpanningAndGetParent(scheduler, cube, new long[] { quiz });
-
- // generate 7d
- System.out.println("Spanning for 7D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 255 });
- // generate 6d
- System.out.println("Spanning for 6D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 });
- // generate 5d
- System.out.println("Spanning for 5D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 });
- // generate 4d
- System.out.println("Spanning for 4D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 129, 130, 132, 248 });
- // generate 3d
- System.out.println("Spanning for 3D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 184, 240 });
- // generate 2d
- System.out.println("Spanning for 2D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 176, 224 });
- // generate 1d
- System.out.println("Spanning for 1D Cuboids");
- testSpanningAndGetParent(scheduler, cube, new long[] { 160, 192 });
- // generate 0d
- }
-
- @Test
- public void testGetSpanningCuboid() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(248);
-
- assertEquals("[]", spnanningCuboids.toString());
- }
-
- @Test
- public void testGetCardinality() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- assertEquals(0, scheduler.getCardinality(0));
- assertEquals(7, scheduler.getCardinality(127));
- assertEquals(1, scheduler.getCardinality(1));
- assertEquals(1, scheduler.getCardinality(8));
- assertEquals(6, scheduler.getCardinality(126));
- }
-
- @Test
- public void testCuboidGeneration1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidGeneration2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidGeneration3() {
- CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
- CuboidCLI.simulateCuboidGeneration(cube);
- }
-
- @Test
- public void testCuboidCounts1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- int[] counts = CuboidCLI.calculateAllLevelCount(cube);
- printCount(counts);
- assertArrayEquals(new int[] { 1, 4, 6, 6, 4, 4, 2, 0 }, counts);
- }
-
- @Test
- public void testCuboidCounts2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- CuboidCLI.calculateAllLevelCount(cube);
- int[] counts = CuboidCLI.calculateAllLevelCount(cube);
- printCount(counts);
- assertArrayEquals(new int[] { 1, 4, 7, 8, 7, 4 }, counts);
- }
-
- private String sortToString(Collection<Long> longs) {
- ArrayList<Long> copy = new ArrayList<Long>(longs);
- Collections.sort(copy);
- return copy.toString();
- }
-
- public CubeDescManager getCubeDescManager() {
- return CubeDescManager.getInstance(getTestConfig());
- }
-
- private void printCount(int[] counts) {
- int sum = 0;
- for (int c : counts)
- sum += c;
- System.out.println(sum + " = " + Arrays.toString(counts));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
deleted file mode 100644
index 7109414..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class CuboidTest extends LocalFileMetadataTestCase {
-
- private long toLong(String bin) {
- return Long.parseLong(bin, 2);
- }
-
- public CubeDescManager getCubeDescManager() {
- return CubeDescManager.getInstance(getTestConfig());
- }
-
- private CubeDesc getTestKylinCubeII() {
- return getCubeDescManager().getCubeDesc("test_kylin_ii");
- }
-
- private CubeDesc getTestKylinCubeWithoutSeller() {
- return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_desc");
- }
-
- private CubeDesc getTestKylinCubeWithSeller() {
- return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_desc");
- }
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testIsValid() {
-
- CubeDesc cube = getTestKylinCubeWithSeller();
-
- // base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
-
- // mandatory column
- assertEquals(false, Cuboid.isValid(cube, toLong("011111110")));
- assertEquals(false, Cuboid.isValid(cube, toLong("100000000")));
-
- // zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("111111000")));
-
- // aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("110000111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("110111000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("111110111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110001")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111110100")));
- assertEquals(false, Cuboid.isValid(cube, toLong("110000100")));
- }
-
- @Test
- public void testCuboid1() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- Cuboid cuboid;
-
- cuboid = Cuboid.findById(cube, 0);
- assertEquals(toLong("10000001"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, 1);
- assertEquals(toLong("10000001"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("00000010"));
- assertEquals(toLong("10000010"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("00100000"));
- assertEquals(toLong("10100000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("01001000"));
- assertEquals(toLong("11111000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("01000111"));
- assertEquals(toLong("11111111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("11111111"));
- assertEquals(toLong("11111111"), cuboid.getId());
- }
-
- @Test
- public void testIsValid2() {
- CubeDesc cube = getTestKylinCubeWithoutSeller();
- try {
- assertEquals(false, Cuboid.isValid(cube, toLong("111111111")));
- fail();
- } catch (IllegalArgumentException ex) {
- // expected
- }
-
- // base
- assertEquals(false, Cuboid.isValid(cube, 0));
- assertEquals(true, Cuboid.isValid(cube, toLong("11111111")));
-
- // aggregation group & zero tail
- assertEquals(true, Cuboid.isValid(cube, toLong("10000111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("11001111")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000001")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10000101")));
-
- // hierarchy
- assertEquals(true, Cuboid.isValid(cube, toLong("10100000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10110000")));
- assertEquals(true, Cuboid.isValid(cube, toLong("10111000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10001000")));
- assertEquals(false, Cuboid.isValid(cube, toLong("10011000")));
- }
-
- @Test
- public void testCuboid2() {
- CubeDesc cube = getTestKylinCubeWithSeller();
- Cuboid cuboid;
-
- cuboid = Cuboid.findById(cube, 0);
- assertEquals(toLong("100111000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, 1);
- assertEquals(toLong("100000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("010"));
- assertEquals(toLong("100000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("0100000"));
- assertEquals(toLong("100100000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("1001000"));
- assertEquals(toLong("101111000"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("1000111"));
- assertEquals(toLong("101000111"), cuboid.getId());
-
- cuboid = Cuboid.findById(cube, toLong("111111111"));
- assertEquals(toLong("111111111"), cuboid.getId());
- }
-
- public void testII() {
- CubeDesc cube = getTestKylinCubeII();
- assertEquals(toLong("111111111"), Cuboid.getBaseCuboidId(cube));
- assertEquals(true, Cuboid.isValid(cube, toLong("111111111")));
- assertEquals(false, Cuboid.isValid(cube, toLong("111111011")));
- assertEquals(false, Cuboid.isValid(cube, toLong("101011011")));
- assertEquals(false, Cuboid.isValid(cube, toLong("000000000")));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
deleted file mode 100644
index 51743c0..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testDecodeWithoutSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
-
- rowKeyDecoder.decode(key);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
-
- assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString());
-
- }
-
- @Test
- public void testDecodeWithSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
-
- rowKeyDecoder.decode(key);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
-
- assertEquals("[SELLER_ID, CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[10000000, 2012-01-02, 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());
-
- }
-
- @Test
- public void testEncodeAndDecodeWithUtf8() throws IOException {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
- CubeDesc cubeDesc = cube.getDescriptor();
-
- byte[][] data = new byte[8][];
- data[0] = Bytes.toBytes("2012-12-15");
- data[1] = Bytes.toBytes("11848");
- data[2] = Bytes.toBytes("Health & Beauty");
- data[3] = Bytes.toBytes("Fragrances");
- data[4] = Bytes.toBytes("Women");
- data[5] = Bytes.toBytes("刊登格式测试");// UTF-8
- data[6] = Bytes.toBytes("0");
- data[7] = Bytes.toBytes("15");
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
-
- byte[] encodedKey = rowKeyEncoder.encode(data);
- assertEquals(30, encodedKey.length);
-
- RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
- rowKeyDecoder.decode(encodedKey);
- List<String> names = rowKeyDecoder.getNames(null);
- List<String> values = rowKeyDecoder.getValues();
- assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
- assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
- }
-}
[27/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv b/examples/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv
deleted file mode 100644
index 441bf62..0000000
--- a/examples/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv
+++ /dev/null
@@ -1,144 +0,0 @@
-48027,Coins & Paper Money:Paper Money: World:Asia:Japan,0,,,,10,Coins,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11116,Coins & Paper Money,3411,8008,48027,48027,48027,48027,Paper Money: World,Asia,Japan,Japan,Japan,Japan,262144,N,-999,Coins,5837,3069,48027,10,1,Collectibles
-164261,Jewelry & Watches:Fashion Jewelry:Earrings:Dangle & Chandelier:Sterling Silver (w/o Stone),0,,,,17,JewelrGemWatches,2008-08-12,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50647,50647,50647,50647,50647,Fashion Jewelry,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,2928,1873,50647,17,1,Fashion
-82494,BookComics & Mags:Comic Books:Modern (1980-now):Superhero:Sandman,15,,,,3,Books,2008-06-03,2010-09-13 07:05:54,USER_X,USER_Y,267,BookMagazines,63,63,63,63,63,63,NULL,Comic Books,Comic Books,Comic Books,Comic Books,Comic Books,262144,N,-999,Books,-999,-999,63,3,1,Media
-66767,Home & Lifestyle:Pet Supplies:Dog Supplies:Dog Apparel:Dresses,15,,,,16,Home & Garden,2008-06-03,2011-06-14 07:48:40,USER_X,USER_Y,11700,Home & Garden,1281,20742,1283,1283,1283,1283,NULL,Dogs,Other Supplies,Other Supplies,Other Supplies,Other Supplies,262144,N,-999,Home Improvement,-999,-999,1283,16,1,Home & Garden
-152801,Jewelry & Watches:Earrings:Studs:DiamonEnhanced Natural:PrincesSquare,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50647,50647,50647,50647,50647,NULL,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,2928,1873,50647,17,1,Fashion
-43398,Home & Garden:Food & Beverages:Cheese & Crackers,0,,,,36,Home & Living,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,11700,Home & Garden,14308,43398,43398,43398,43398,43398,NULL,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,262144,N,-999,Home Furnishing,-999,-999,43398,36,1,Home & Garden
-95173,Health & Beauty:Bath & Body:Bath SetKits:Avon,0,,,,37,Health & Beauty,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,26395,Health & Beauty,11838,67391,67391,67391,67391,67391,Bath & Body,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,262144,N,-999,Health & Beauty,2919,1868,67391,37,1,Fashion
-158666,ToyHobbies:Action Figures:Anime & Manga,15,,,,28,Toys,2012-09-11,2012-09-11 20:27:22,USER_X,USER_Y,220,ToyHobbies,246,158666,158666,158666,158666,158666,Action Figures,Anime & Manga,Anime & Manga,Anime & Manga,Anime & Manga,Anime & Manga,262144,N,-999,Toys,-999,-999,158666,28,1,Collectibles
-12688,eBay Premier:Books & Manuscripts:Books: Other:Icollector,0,,2038,,57,Sothebys,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,2038,eBay Premier,1680,1681,12688,12688,12688,12688,Books & Manuscripts,Books: Other,Icollector,Icollector,Icollector,Icollector,60,N,,Sothebys,-999,-999,0,57,1,Lifestyle
-103324,Clothing & Accessories:Women:Shoes:Casual SneakerSkate:Sizes 10+,15,,,,9,Clothing & Accessories,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoeAccessories,3034,63889,63889,63889,63889,63889,Women's Shoes,Mixed Items,Mixed Items,Mixed Items,Mixed Items,Mixed Items,262144,N,-999,Clothing & Accessories,2441,1634,-99,9,1,Fashion
-108782,Vehicle Parts & Accessories:CaTruck Parts:Car Care & Cleaning:Air Fresheners,15,,,,8,Auto Parts,2012-09-11,2013-09-10 16:52:46,USER_X,USER_Y,131090,Vehicle Parts & Accessories,6030,72200,108782,108782,108782,108782,CaTruck Parts,Car Care & Cleaning,Air Fresheners,Air Fresheners,Air Fresheners,Air Fresheners,262144,N,-999,Auto - Parts,-999,-999,108782,8,1,Parts & Accessories
-80287,Computers & Networking:Software:Business & Productivity:Voice Recognition,0,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers/Tablets & Networking,18793,3768,3768,3768,3768,3768,Software,Office & Business,Office & Business,Office & Business,Office & Business,Office & Business,262144,N,-999,Computers,5893,3093,3768,12,1,Electronics
-140746,eBay Motors:Parts & Accessories:Vintage Car & Truck Parts:WheelTire& Hub Caps:Tires,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,10073,140744,140746,140746,140746,Parts & Accessories,Vintage Car & Truck Parts,WheelTire& Hub Caps,Tires,Tires,Tires,262144,N,-999,Auto - Parts,3517,1794,140746,8,1,Parts & Accessories
-87118,Sporting Goods:Paintball:Barrels:Intimidator,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,159043,16045,16046,16046,16046,16046,Outdoor Sports,Paintball,Barrels,Barrels,Barrels,Barrels,0,N,-999,Sporting Goods,2919,1868,16046,24,1,Lifestyle
-25147,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:Baseball-MLB:Philadelphia Phillies:Other Items,0,,888,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:51:44,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24410,24410,24410,24410,24410,Fan Apparel & Souvenirs,Baseball-MLB,Baseball-MLB,Baseball-MLB,Baseball-MLB,Baseball-MLB,262144,N,-999,Sports Memorabilia,2884,1845,24410,25,1,Collectibles
-170302,Crafts:Embroidery:Design CDs:Other Design Themes,15,,,,15,Crafts,2009-09-21,2010-09-13 07:13:34,USER_X,USER_Y,14339,Crafts,28141,41383,41383,41383,41383,41383,Embroidery,Design CDs,Design CDs,Design CDs,Design CDs,Design CDs,262144,N,-999,Hobbies & Crafts,-999,-999,41383,15,1,Home & Garden
-53064,Business & Industrial:Agriculture & Forestry:Antique Tractors & Equipment:Tractors:Massey HarriFerguson,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,177641,177642,177642,177642,177642,177642,Heavy Equipment,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,262144,N,-999,Business (Office & Industrial),-999,-999,177642,4,1,Business & Industrial
-132939,Jewelry & Watches:Designer Brands:John Atencio,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,499,499,499,499,499,Fashion Jewelry,Other,Other,Other,Other,Other,262144,N,-999,JewelrGemWatches,2928,1873,499,17,1,Fashion
-113593,Mobiles & Phones:Mobile Phones:Samsung:Samsung D500,15,,,,40,Cell Phones & Portable Electro,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,15032,Phones,9355,9355,9355,9355,9355,9355,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,262144,N,-999,Cell Phones & Accessories,2246,1609,9355,40,1,Electronics
-34273,eBay Motors:Parts & Accessories:Motorcycle:Frame & Suspension:Frames,100,,,,8,Auto - Parts,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,6000,eBay Motors,6028,10063,34272,34273,34273,34273,Parts & Accessories,Motorcycle,Frame & Suspension,Frames,Frames,Frames,0,N,-999,Auto - Parts,-999,-999,0,8,1,Parts & Accessories
-106340,Home & Lifestyle:Tools & Building Materials:Hand Tools:Socket Sets,15,,,,16,Home & Garden,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,2032,29515,29515,29515,29515,29515,Gardening,Hand Tools,Hand Tools,Hand Tools,Hand Tools,Hand Tools,262144,N,-999,Home Improvement,-999,-999,29515,16,1,Home & Garden
-150265,Baby & Maternity:Baby Clothing & Shoes:Boys:Size 00:Outfits & Sets:Bonds,15,,,,2,Baby,2008-06-03,2011-04-05 08:25:25,USER_X,USER_Y,2984,Baby,3082,147317,147333,147333,147333,147333,Baby Clothing,Boys,OutfitSets,OutfitSets,OutfitSets,OutfitSets,0,N,-999,Baby,3090,1947,147333,2,1,Home & Garden
-24760,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:Hockey-NHL:San Jose Sharks:CapHats,0,,888,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:51:44,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24510,24510,24510,24510,24510,Fan Apparel & Souvenirs,Hockey-NHL,Hockey-NHL,Hockey-NHL,Hockey-NHL,Hockey-NHL,262144,N,-999,Sports Memorabilia,2884,1845,24510,25,1,Collectibles
-37831,Collectibles:Advertising:Merchandise & Memorabilia:Advertising-Print:1910-19,0,,,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,34,13623,37828,37831,37831,37831,Advertising,Merchandise & Memorabilia,Advertising-Print,1910-19,1910-19,1910-19,262144,N,-999,Collectibles,-999,-999,37831,11,1,Collectibles
-1120,Books:First Editions:Other,3,,267,Other,3,BookComics & Magazines,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,267,Books,2231,1120,1120,1120,1120,1120,First Editions,Other,Other,Other,Other,Other,0,N,-999,Books,,,0,3,1,Media
-43972,eBay Motors:Parts & Accessories:ATV Parts:Other,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,43962,43972,43972,43972,43972,Parts & Accessories,ATV Parts,Other,Other,Other,Other,262144,N,-999,Auto - Parts,5411,2941,43972,8,1,Parts & Accessories
-166013,Computers:Apple AccessorieParts:TV Tuners,15,,,,12,Computers,2008-09-09,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers,175673,3761,3761,3761,3761,3761,Computer Components & Parts,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,262144,N,-999,Computers,-999,-999,3761,12,1,Electronics
-15568,Baby:Baby Clothing:Unisex:Sleepwear,15,,,,2,Baby,2008-06-03,2011-04-05 08:25:25,USER_X,USER_Y,2984,Baby,3082,163397,163400,163400,163400,163400,Baby Clothing,Unisex,Sleepwear,Sleepwear,Sleepwear,Sleepwear,0,N,-999,Baby,3090,1947,163400,2,1,Home & Garden
-103178,Clothing & Accessories:Women:HandbagBags:Evening Bags,15,,,,9,Clothing & Accessories,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoeAccessories,169291,169291,169291,169291,169291,169291,Women's Bags,Women's Bags,Women's Bags,Women's Bags,Women's Bags,Women's Bags,262144,N,-999,Clothing & Accessories,2487,1098,-99,9,1,Fashion
-2023,Sporting Goods:Team Sports:Basketball:Other,0,,888,Sporting Goods,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,159049,21194,2023,2023,2023,2023,Team Sports,Basketball,Other,Other,Other,Other,262144,N,-999,Sporting Goods,-999,-999,2023,24,1,Lifestyle
-94847,Consumer Electronics:Car Electronics:Car Video:Car Video Monitors Only:Headrest Monitors:6-6.9 inches,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,3270,48604,48606,48606,48606,48606,Vehicle Electronics & GPS,Car Video,Car Monitors w/o Player,Car Monitors w/o Player,Car Monitors w/o Player,Car Monitors w/o Player,262144,N,-999,Consumer Electronics - Other,2920,1869,48606,13,1,Electronics
-15868,Real Estate:Land:Central U.S.:Illinois,0,,10542,,23,Real Estate,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,10542,Real Estate,15841,15841,15841,15841,15841,15841,Land,Land,Land,Land,Land,Land,0,N,-999,Real Estate,-999,-999,15841,23,1,Real Estate
-32876,Home & Garden:Home Improvement:Plumbing & Fixtures:Toilet Paper Holders-Mounted,0,,,,36,Home & Living,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,159907,20601,32876,32876,32876,32876,Home Improvement,Plumbing & Fixtures,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,262144,N,-999,Home Furnishing,-999,-999,32876,36,1,Home & Garden
-62179,Sporting Goods:Skiing & Snowboarding:Apparel:Ski Suits:Women,0,,,,9,Apparel,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,137085,137085,137085,137085,137085,Women's Clothing,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,0,N,-999,Clothing & Accessories,2919,1868,137085,9,1,Fashion
-45333,ClothinShoes & Accessories:Women's Shoes:Flats & Oxfords,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,45333,45333,45333,45333,45333,Women's Shoes,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,262144,N,-999,Clothing & Accessories,-999,-999,45333,9,1,Fashion
-50677,Jewelry & Watches:Fashion Jewelry:Pins & Brooches,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50677,50677,50677,50677,50677,Fashion Jewelry,Pins & Brooches,Pins & Brooches,Pins & Brooches,Pins & Brooches,Pins & Brooches,262144,N,-999,JewelrGemWatches,-999,-999,50677,17,1,Fashion
-33038,Musical Instruments:Guitars:Electric:ESP,15,,,,30,Musical Instruments,2008-06-03,2010-06-14 07:48:40,USER_X,USER_Y,619,Musical Instruments,166070,33034,33034,33034,33034,33034,Instruments,Guitars (Electric),Guitars (Electric),Guitars (Electric),Guitars (Electric),Guitars (Electric),262144,N,-999,Musical Instruments,-999,-999,33034,30,1,Lifestyle
-156614,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:NASCAR:Revell:Dale Jarrett,0,,,,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,171127,762,762,762,762,Diecast & Toy Vehicles,Cars: RacinNASCAR,NASCAR,NASCAR,NASCAR,NASCAR,262144,N,-999,Toys,1740,1380,762,28,1,Collectibles
-106246,Health & Beauty:Hair Care:Conditioner:Biolage,0,,,,37,Health & Beauty,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,26395,Health & Beauty,11854,177661,177661,177661,177661,177661,Hair Care & Styling,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,262144,N,-999,Health & Beauty,2919,1868,177661,37,1,Fashion
-20865,Sporting Goods:Soccer:Apparel & Footwear:Jerseys:Other,0,,888,,9,Apparel,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,137084,137084,137084,137084,137084,Men's Clothing,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,0,N,-999,Clothing & Accessories,2919,1868,137084,9,1,Fashion
-15115,Entertainment:Video Games:Games:Sega Dreamcast:Sports,0,,293,,32,Video Games,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Video Games,Video Games,Video Games,Video Games,Video Games,Video Games,0,N,-999,Video Games,-999,-999,139973,32,1,Electronics
-3838,JewelrGemstones:Jewelry:Charms:Plated,0,,281,Other,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,140944,140944,140944,140944,140944,Fashion Jewelry,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,0,N,-999,JewelrGemWatches,-999,-999,140944,17,1,Fashion
-759,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:Franklin Mint:Modern (1970-Now),0,,220,Other,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,19036,223,223,223,223,Diecast & Toy Vehicles,CarTrucks & Vans,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,262144,N,-999,Toys,1740,1380,223,28,1,Collectibles
-61323,Consumer Electronics:A/V Accessories & Cables:Remote Controls:Universal Remotes,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,32852,14961,61312,61312,61312,61312,TVideo & Home Audio,TVideo & Audio Accessories,Remote Controls,Remote Controls,Remote Controls,Remote Controls,262144,N,-999,Consumer Electronics - Other,5344,2248,61312,13,1,Electronics
-121153,Baby:Nursery Decor:Night Lights,0,,,,2,Baby,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,2984,Baby,66697,121153,121153,121153,121153,121153,Nursery Decor,Night Lights,Night Lights,Night Lights,Night Lights,Night Lights,262144,N,-999,Baby,-999,-999,121153,2,1,Home & Garden
-88750,Consumer Electronics:Car Electronics:RadaLaser Detectors:Whistler:17174 Series,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,3270,14935,14935,14935,14935,14935,Vehicle Electronics & GPS,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,262144,N,-999,Consumer Electronics - Other,2920,1869,14935,13,1,Electronics
-161567,Computers:Laptop Accessories:Batteries:Hewlett Packard,15,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers,31530,14295,14295,14295,14295,14295,Laptop & Desktop Accessories,Laptop Batteries,Laptop Batteries,Laptop Batteries,Laptop Batteries,Laptop Batteries,262144,N,-999,Computers,3699,2194,14295,12,1,Electronics
-113802,Lots More...:Metaphysical:Herbs,15,,,,99,Everything Else,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,99,Lots More...,19266,113802,113802,113802,113802,113802,Metaphysical,Herbs,Herbs,Herbs,Herbs,Herbs,262144,N,-999,Everything Else,-999,-999,113802,99,1,Lifestyle
-15808,Clothing & Accessories:Women:Plus Sizes:Tops:Other Tops,15,,,,9,Clothing & Accessories,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoeAccessories,15724,53159,53159,53159,53159,53159,Women's Clothing,Tops & Blouses,Tops & Blouses,Tops & Blouses,Tops & Blouses,Tops & Blouses,262144,N,-999,Clothing & Accessories,2264,1551,53159,9,1,Fashion
-174053,Vehicle Parts & Accessories:Car Parts:External & Body Parts:SunrooConvertible & Hardtop,3,,,,8,Auto - Parts,2011-08-23,2013-04-28 20:37:19,USER_X,USER_Y,131090,Vehicle Parts & Accessories,9884,14768,174053,174053,174053,174053,Car Parts,External & Body Parts,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,262144,N,-999,Auto - Parts,4075,2390,174053,8,1,Parts & Accessories
-2635,Toys & Hobbies:Toy Soldiers:1970-Now:Plastic,0,,220,Other,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,2631,2638,2638,2638,2638,2638,Toy Soldiers,1970-Now,1970-Now,1970-Now,1970-Now,1970-Now,262144,N,-999,Toys,2919,1868,2638,28,1,Collectibles
-1161,DVFilm & TV:Other Formats:Videos: NTSC (US):ActioAdventure,3,,267,Other,18,DVDFilm & TV,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11232,DVFilm & TV,380,1151,1161,1161,1161,1161,Other Formats,Videos: NTSC (US),ActioAdventure,ActioAdventure,ActioAdventure,ActioAdventure,0,N,-999,DVDs & Movies,-999,-999,0,18,1,Media
-64076,Computers & Networking:Networking:Switches:3Com:SuperStack,0,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers/Tablets & Networking,175698,175706,51268,51268,51268,51268,Enterprise NetworkinServers,Switches & Hubs,Network Switches,Network Switches,Network Switches,Network Switches,262144,N,-999,Computers,2919,1868,51268,12,1,Electronics
-33977,Crafts:Scrapbooking:Albums:12x12 Albums,15,,,,15,Crafts,2008-06-03,2010-06-14 07:56:25,USER_X,USER_Y,14339,Crafts,31253,33875,33875,33875,33875,33875,Scrapbooking,Albums,Albums,Albums,Albums,Albums,262144,N,-999,Hobbies & Crafts,-999,-999,33875,15,1,Home & Garden
-31673,Sports:Memorabilia:Racing-NASCAR:Plaques,0,,,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:48:39,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,46156,46156,46156,46156,46156,Fan Apparel & Souvenirs,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,0,N,-999,Sports Memorabilia,-999,-999,46156,25,1,Collectibles
-174106,Vehicle Parts & Accessories:Car Parts:Transmission & Drivetrain:Sensors,3,,,,8,Auto - Parts,2011-08-23,2013-04-28 20:37:19,USER_X,USER_Y,131090,Vehicle Parts & Accessories,9884,33726,174106,174106,174106,174106,Car Parts,Transmission & Drivetrain,Sensors,Sensors,Sensors,Sensors,262144,N,-999,Auto - Parts,4075,2390,174106,8,1,Parts & Accessories
-26249,Business & Industrial:Printing & Graphic Arts:Commercial Printing Presses:Sheet-fed Offset Presses,0,,9428,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,26238,26247,26249,26249,26249,26249,Printing & Graphic Arts,Commercial Printing Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,262144,N,-999,Business (Office & Industrial),-999,-999,26249,4,1,Business & Industrial
-159184,Sporting Goods:Winter Sports:Snowboarding:Stomp Pads,0,,,,24,Sporting Goods,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,888,Sporting Goods,36259,21247,159184,159184,159184,159184,Winter Sports,Snowboarding,Stomp Pads,Stomp Pads,Stomp Pads,Stomp Pads,262144,N,-999,Sporting Goods,-999,-999,159184,24,1,Lifestyle
-10058,Tickets & Travel:Tickets:Sporting Events:Wrestling,3,,11730,,27,Tickets,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,1305,Events Tickets,1306,1306,1306,1306,1306,1306,Other Tickets,Other Tickets,Other Tickets,Other Tickets,Other Tickets,Other Tickets,262144,N,-999,Tickets,2937,1879,1306,27,1,Lifestyle
-48904,ClothinShoes & Accessories:Vintage:Women's Vintage Clothing:1977-89 (PunNew Wav80s):Sweaters,0,,,,9,Apparel,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoes & Accessories,175759,175762,52451,48904,48904,48904,Vintage,Women's Vintage Clothing,1977-89 (PunNew Wav80s),Sweaters,Sweaters,Sweaters,262144,N,-999,Clothing & Accessories,-999,-999,48904,9,1,Fashion
-145970,Toys & Hobbies:ModelKits:Automotive:Vintage:Plymouth,0,,,,28,Toys,2008-06-03,2008-10-08 07:18:40,USER_X,USER_Y,220,Toys & Hobbies,1188,2580,2585,2585,2585,2585,Models & Kits,Automotive,Vintage,Vintage,Vintage,Vintage,262144,N,-999,Toys,2974,1905,2585,28,1,Collectibles
-963,ClothinShoes & Accessories:Vintage:Clothing (Pre-1980):Women:ClothinShoes & Accessories:Shoes,0,,1,Other,9,Apparel,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoes & Accessories,175759,74976,48561,48561,48561,48561,Vintage,Women's Vintage Shoes,Other,Other,Other,Other,0,N,-999,Clothing & Accessories,-999,-999,48561,9,1,Fashion
-118687,Health & Beauty:Fragrances:Women's Fragrances:Benetton,3,,,,37,Health & Beauty,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,26395,Health & Beauty,26396,11848,11848,11848,11848,11848,Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,262144,N,-999,Health & Beauty,6778,3189,11848,37,1,Fashion
-20886,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:NASCAR:Action/ Motorsports Authentic:Darrell Waltrip,0,,220,,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,171127,762,762,762,762,Diecast & Toy Vehicles,Cars: RacinNASCAR,NASCAR,NASCAR,NASCAR,NASCAR,262144,N,-999,Toys,1740,1380,762,28,1,Collectibles
-148324,Mobiles & Phones:Mobile Accessories:Covers / Faceplates:Sony Ericsson:W900i,15,,,,31,Phones,2008-06-03,2012-09-18 00:08:03,USER_X,USER_XIANZHU,15032,Phones,9394,20349,20349,20349,20349,20349,Mobile Accessories,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,262144,N,-999,Telecomm,-999,-999,20349,31,1,Electronics
-139255,JewellerGemWatches:Fine Jewellery:Earrings:Gemstone - Created:Lapis,15,,,,17,JewelrGemWatches,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,281,Jewellery & Watches,4196,10985,10985,10985,10985,10985,Fine Jewellery,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,-999,-999,-99,17,1,Fashion
-20213,Collectibles:Postcards:US StateCities & Towns:Maine,0,,1,,11,Collectibles,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,1,Collectibles,914,20193,20213,20213,20213,20213,Postcards,US StateCities & Towns,Maine,Maine,Maine,Maine,262144,N,-999,Collectibles,-999,-999,20213,11,1,Collectibles
-32996,Movies:Television Memorabilia:Clippings,15,,,,42,Entertainment Memorabilia,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11232,Movies,1424,32996,32996,32996,32996,32996,Television Memorabilia,Clippings,Clippings,Clippings,Clippings,Clippings,262144,N,-999,Entertainment Memorabilia,-999,-999,32996,42,1,Collectibles
-99985,Collectibles:Trading Cards:Sci-FFantasy:Star Wars:Sets:Other,0,,,,11,Collectibles,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,1,Collectibles,868,37897,100899,99980,99985,99985,Trading Cards,Sci-FFantasy,Star Wars,Sets,Other,Other,262144,N,-999,Collectibles,-999,-999,99985,11,1,Collectibles
-67703,Jewellery & Watches:Jewellery Boxes & Supplies:Jewellery Display,3,,,,17,Jewellery & Watches,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,281,Jewellery & Watches,10321,67703,67703,67703,67703,67703,Jewellery Boxes & Supplies,Jewellery Display,Jewellery Display,Jewellery Display,Jewellery Display,Jewellery Display,262144,N,-999,JewelrGemWatches,4435,2261,67703,17,1,Fashion
-164,Computers/Tablets & Networking:Computer Components & Parts:CPUProcessors,0,,160,Computer Hardware,12,Computers,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,164,164,164,164,164,Computer Components & Parts,CPUProcessors,CPUProcessors,CPUProcessors,CPUProcessors,CPUProcessors,262144,N,-999,Computers,5730,2011,164,12,1,Electronics
-279,BookMagazines:Children's Books,15,,267,Other,3,Books,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,267,BookMagazines,1093,1093,1093,1093,1093,1093,Children's Books,Children's Books,Children's Books,Children's Books,Children's Books,Children's Books,262144,N,-999,Books,1415,1128,1093,3,1,Media
-65,Collectibles:Comics:Platinum Age (1897-1937),0,,1,Other,11,Collectibles,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,1,Collectibles,63,65,65,65,65,65,Comics,Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),262144,N,-999,Collectibles,6014,2967,65,11,1,Collectibles
-24541,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:College-NCAA,0,,,,25,Sports Memorabilia,2010-02-08,2013-07-16 08:23:09,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24541,24541,24541,24541,24541,Fan Apparel & Souvenirs,College-NCAA,College-NCAA,College-NCAA,College-NCAA,College-NCAA,262144,N,-999,Sports Memorabilia,7721,2482,24541,25,1,Collectibles
-23446,Mode & Accessoires:Chaussures de femme:Sandales & Sandalettes:Taille 37-38,23,,,,9,Vtements et Accessoires,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,11450,Mode & Accessoires,35107,23418,23446,23446,23446,23446,Chaussures de femme,Sandales & Sandalettes,Taille 37-38,Taille 37-38,Taille 37-38,Taille 37-38,262144,N,-999,Clothing & Accessories,2438,1634,23446,9,1,Fashion
-130,Collectibles:Transportation:Railroadiana & Trains:Other,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,417,1444,130,130,130,130,Transportation,Railroadiana & Trains,Other,Other,Other,Other,262144,N,-999,Collectibles,-999,-999,130,11,1,Collectibles
-216,Sports MeCards & Fan Shop:Cards:Hockey,0,,11800,Other,25,Sports Memorabilia,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,212,216,216,216,216,216,Cards,Hockey,Hockey,Hockey,Hockey,Hockey,262144,N,-999,Sports Memorabilia,3636,2152,216,25,1,Collectibles
-10866,Collectibles:Animals:Farm & Countryside:Pigs,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,1335,45602,10866,10866,10866,10866,Animals,Farm & Countryside,Pigs,Pigs,Pigs,Pigs,262144,N,-999,Collectibles,-999,-999,10866,11,1,Collectibles
-57013,Business & Industrial:MRO & Industrial Supply:Pumps & Plumbing:Pumps:Pump Accessories & Parts,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,1266,109619,42928,57013,57013,57013,MRO & Industrial Supply,Pumps & Plumbing,Pumps,Pump Accessories & Parts,Pump Accessories & Parts,Pump Accessories & Parts,262144,N,-999,Business (Office & Industrial),-999,-999,57013,4,1,Business & Industrial
-15687,ClothinShoes & Accessories:Men's Clothing:T-Shirts,0,,11450,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,15687,15687,15687,15687,15687,Men's Clothing,T-Shirts,T-Shirts,T-Shirts,T-Shirts,T-Shirts,262144,N,2515,Clothing & Accessories,7692,1115,15687,9,1,Fashion
-80135,Computers/Tablets & Networking:DriveStorage & Blank Media:Blank Media & Accessories:CDVD & Blu-ray Discs,0,,,,12,Computers,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,165,80133,80135,80135,80135,80135,DriveStorage & Blank Media,Blank Media & Accessories,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,262144,N,-999,Computers,3374,2018,80135,12,1,Electronics
-31519,Computers/Tablets & Networking:Laptop & Desktop Accessories:Laptop Cases & Bags,0,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,31530,31519,31519,31519,31519,31519,Laptop & Desktop Accessories,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,262144,N,-999,Computers,6404,3189,31519,12,1,Electronics
-175750,Home & Garden:Bedding:Blankets & Throws,0,,,,36,Home & Living,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,11700,Home & Garden,20444,175750,175750,175750,175750,175750,Bedding,Blankets & Throws,Blankets & Throws,Blankets & Throws,Blankets & Throws,Blankets & Throws,262144,N,-999,Home Furnishing,-999,-999,175750,36,1,Home & Garden
-63864,ClotheShoes & Accessories:Women's Clothing:Skirts,3,,,,9,Clothing & Accessories,2009-06-15,2012-06-19 21:15:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,15724,63864,63864,63864,63864,63864,Women's Clothing,Skirts,Skirts,Skirts,Skirts,Skirts,262144,N,-999,Clothing & Accessories,6334,1551,63864,9,1,Fashion
-13836,Collectibles:Decorative Collectibles:Spoons,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,13836,13836,13836,13836,13836,Decorative Collectibles,Spoons,Spoons,Spoons,Spoons,Spoons,262144,N,-999,Collectibles,-999,-999,13836,11,1,Collectibles
-63861,ClothinShoes & Accessories:Women's Clothing:Dresses,0,,,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,63861,63861,63861,63861,63861,Women's Clothing,Dresses,Dresses,Dresses,Dresses,Dresses,262144,N,-999,Clothing & Accessories,1263,1112,63861,9,1,Fashion
-150047,Crafts:Jewellery Making:Findings:Bails,3,,,,15,Hobbies & Crafts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,14339,Crafts,150045,150046,150047,150047,150047,150047,Jewellery Making,Findings,Bails,Bails,Bails,Bails,262144,N,-999,Hobbies & Crafts,4618,2740,150047,15,1,Home & Garden
-95672,ClotheShoes & Accessories:Women's Shoes:Trainers,3,,,,9,Clothing & Accessories,2011-04-05,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,3034,95672,95672,95672,95672,95672,Women's Shoes,Trainers,Trainers,Trainers,Trainers,Trainers,262144,N,-999,Clothing & Accessories,2314,1634,95672,9,1,Fashion
-57990,ClotheShoes & Accessories:Men's Clothing:Casual Shirts & Tops,3,,,,9,Clothing & Accessories,2009-06-15,2012-06-19 21:15:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,1059,57990,57990,57990,57990,57990,Men's Clothing,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,262144,N,-999,Clothing & Accessories,6331,1623,57990,9,1,Fashion
-75708,Toys & Games:Action Figures:TMovies & Video Games,3,,,,28,Toys & Games,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,220,Toys & Games,246,75708,75708,75708,75708,75708,Action Figures,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,262144,N,-999,Toys,-999,-999,75708,28,1,Collectibles
-4943,Toys & Hobbies:Diecast & Toy Vehicles:CarTrucks & Vans:Plastic:Vintage Manufacture (Pre-1970),0,,220,Other,28,Toys,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,220,Toys & Hobbies,222,19036,19060,4943,4943,4943,Diecast & Toy Vehicles,CarTrucks & Vans,Plastic,Vintage Manufacture (Pre-1970),Vintage Manufacture (Pre-1970),Vintage Manufacture (Pre-1970),262144,N,-999,Toys,1740,1380,4943,28,1,Collectibles
-40059,Mobile Phones & Communication:Radio Communication Equipment:Parts & Accessories:Other Parts & Accessories,3,,,,33,Consumer Electronics - Audio,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,15032,Mobile Phones & Communication,1500,175734,40059,40059,40059,40059,Radio Communication Equipment,Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,262144,N,-999,Consumer Electronics - Audio,-999,-999,40059,33,1,Electronics
-43479,Cameras & Photo:Film Photography:Other,0,,,,21,Photo,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,625,Cameras & Photo,69323,43479,43479,43479,43479,43479,Film Photography,Other,Other,Other,Other,Other,262144,N,-999,Photo,-999,-999,43479,21,1,Electronics
-11848,Health & Beauty:Fragrances:Women,0,,,,37,Health & Beauty,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,26395,Health & Beauty,26396,11848,11848,11848,11848,11848,Fragrances,Women,Women,Women,Women,Women,262144,N,-999,Health & Beauty,5884,3087,11848,37,1,Fashion
-165888,Jewelry & Watches:Vintage & Antique Jewelry:Costume:RetrVintage 1930s-1980s:PinBrooches,0,,,,17,JewelrGemWatches,2008-09-09,2009-01-12 07:05:17,USER_X,USER_Y,281,Jewelry & Watches,48579,500,58559,165888,165888,165888,Vintage & Antique Jewelry,Costume,RetrVintage 1930s-1980s,PinBrooches,PinBrooches,PinBrooches,262144,N,-999,JewelrGemWatches,-999,-999,165888,17,1,Fashion
-26262,Collectibles:Advertising:Food & Beverage:Distillery:Bacardi,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,34,13587,820,26262,26262,26262,Advertising,Food & Beverage,Distillery,Bacardi,Bacardi,Bacardi,262144,N,-999,Collectibles,-999,-999,26262,11,1,Collectibles
-67698,Business & Industrial:Retail & Services:Jewellery Packaging & Display:Cases & Displays:Bracelet,2,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11890,67694,67697,67698,67698,67698,Retail & Services,Jewellery Packaging & Display,Cases & Displays,Bracelet,Bracelet,Bracelet,262144,N,-999,Business (Office & Industrial),-999,-999,67698,4,1,Business & Industrial
-57990,ClothinShoes & Accessories:Men's Clothing:Casual Shirts,0,,,,9,Apparel,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,57990,57990,57990,57990,57990,Men's Clothing,Casual Shirts,Casual Shirts,Casual Shirts,Casual Shirts,Casual Shirts,262144,N,-999,Clothing & Accessories,-999,-999,57990,9,1,Fashion
-67698,Business & Industrial:Retail & Services:Jewelry Packaging & Display:Cases & Displays:Bracelet,0,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11890,67694,67697,67698,67698,67698,Retail & Services,Jewelry Packaging & Display,Cases & Displays,Bracelet,Bracelet,Bracelet,262144,N,-999,Business (Office & Industrial),-999,-999,67698,4,1,Business & Industrial
-46575,Business & Industrial:Light Equipment & Tools:Air Tools:Air Tool Accessories,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,61573,46573,46575,46575,46575,46575,Light Equipment & Tools,Air Tools,Air Tool Accessories,Air Tool Accessories,Air Tool Accessories,Air Tool Accessories,262144,N,-999,Business (Office & Industrial),-999,-999,46575,4,1,Business & Industrial
-41940,Business & Industrial:Manufacturing & Metalworking:Metalworking Tooling:Toolholding:Indexable Inserts,0,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11804,92084,45017,41940,41940,41940,Manufacturing & Metalworking,Metalworking Tooling,Toolholding,Indexable Inserts,Indexable Inserts,Indexable Inserts,262144,N,-999,Business (Office & Industrial),-999,-999,41940,4,1,Business & Industrial
-51582,ClothinShoes & Accessories:Kids' ClothinShoes & Accs:Girls' Clothing (Sizes 4 & Up):Sweaters,0,,,,9,Apparel,2012-05-01,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,171146,11462,51582,51582,51582,51582,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Sweaters,Sweaters,Sweaters,Sweaters,262144,N,-999,Clothing & Accessories,-999,-999,51582,9,1,Fashion
-80053,Computers/Tablets & Networking:MonitorProjectors & Accs:Monitors,0,,,,12,Computers,2009-09-21,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,162497,80053,80053,80053,80053,80053,MonitorProjectors & Accs,Monitors,Monitors,Monitors,Monitors,Monitors,262144,N,-999,Computers,2982,1911,80053,12,1,Electronics
-31387,Jewellery & Watches:Watches:Wristwatches,3,,,,17,Jewellery & Watches,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,281,Jewellery & Watches,14324,31387,31387,31387,31387,31387,Watches,Wristwatches,Wristwatches,Wristwatches,Wristwatches,Wristwatches,262144,N,-999,JewelrGemWatches,7489,32,31387,17,1,Fashion
-20485,Home & Garden:Inside the Home:Furniture:Dining Room:Other,0,,11700,,36,Home & Living,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11700,Home & Garden,3197,175752,175752,175752,175752,175752,Furniture,Other,Other,Other,Other,Other,262144,N,-999,Home Furnishing,2919,1868,175752,36,1,Home & Garden
-44079,Sporting Goods:Exercise & Fitness:GyWorkout & Yoga:Fitness Equipment:Exercise Mats,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,15273,158932,28064,44079,44079,44079,Exercise & Fitness,GyWorkout & Yoga,Fitness Equipment,Exercise Mats,Exercise Mats,Exercise Mats,262144,N,-999,Sporting Goods,-999,-999,44079,24,1,Lifestyle
-175750,HomFurniture & DIY:Bedding:Blankets,3,,,,16,Home,2012-05-01,2012-09-11 20:26:04,USER_X,USER_Y,11700,HomFurniture & DIY,20444,175750,175750,175750,175750,175750,Bedding,Blankets,Blankets,Blankets,Blankets,Blankets,262144,N,-999,Home Improvement,-999,-999,175750,16,1,Home & Garden
-16509,Toys & Hobbies:Model Railroads & Trains:S Scale:Other,0,,220,,28,Toys,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,220,Toys & Hobbies,479,19143,16509,16509,16509,16509,Model Railroads & Trains,S Scale,Other,Other,Other,Other,262144,N,-999,Toys,-999,-999,16509,28,1,Collectibles
-139973,Video Games & Consoles:Games,3,,,,32,PC & Video Gaming,2011-04-05,2012-09-11 20:26:04,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Games,Games,Games,Games,Games,Games,262144,N,-999,Video Games,1655,1135,139973,32,1,Electronics
-170083,Computers/Tablets & Networking:Computer Components & Parts:Memory (RAM),3,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,170083,170083,170083,170083,170083,Computer Components & Parts,Memory (RAM),Memory (RAM),Memory (RAM),Memory (RAM),Memory (RAM),262144,N,-999,Computers,3369,2042,170083,12,1,Electronics
-16145,Computers/Tablets & Networking:Computer Components & Parts:Other Components & Parts,3,,160,,12,Computers,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,16145,16145,16145,16145,16145,Computer Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,262144,N,-999,Computers,6976,3189,16145,12,1,Electronics
-158798,Toys & Hobbies:Vintage & Antique Toys:Spinning Tops,0,,,,28,Toys,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,220,Toys & Hobbies,717,158798,158798,158798,158798,158798,Vintage & Antique Toys,Spinning Tops,Spinning Tops,Spinning Tops,Spinning Tops,Spinning Tops,262144,N,-999,Toys,-999,-999,158798,28,1,Collectibles
-20485,CasArredamento e Bricolage:Cucina:Altro per cucina,101,,,,36,Mobili per la casa,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11700,CasArredamento e Bricolage,20625,20485,20485,20485,20485,20485,Cucina,Altro per cucina,Altro per cucina,Altro per cucina,Altro per cucina,Altro per cucina,0,N,-999,Home Furnishing,-999,-999,20485,36,1,Home & Garden
-100847,Half Books,0,,,,3,Books,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,100847,Half Books,100847,100847,100847,100847,100847,100847,Half Books,Half Books,Half Books,Half Books,Half Books,Half Books,0,N,-999,Books,-999,-999,100847,3,2,Media
-156356,Collectibles:Postcards:BuildingArchitecture,0,,,,11,Collectibles,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,1,Collectibles,914,156356,156356,156356,156356,156356,Postcards,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,262144,N,-999,Collectibles,-999,-999,156356,11,1,Collectibles
-60606,Collectables:Badges/ Patches:Golly Badges:Robertsons:Modern (Post-1990),3,,,,11,Collectables,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1,Collectables,32615,98793,106160,60606,60606,60606,Badges/ Patches,Golly Badges,Robertsons,Modern (Post-1990),Modern (Post-1990),Modern (Post-1990),262144,N,-999,Collectibles,4390,2640,60606,11,1,Collectibles
-36250,Sporting Goods:Outdoor Sports:Hunting:Decoys:Duck,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,7301,36249,36249,36249,36249,36249,Hunting,Decoys,Decoys,Decoys,Decoys,Decoys,262144,N,-999,Sporting Goods,-999,-999,36249,24,1,Lifestyle
-50508,Cameras & Photo:Camera & Photo Accessories:LCD Hoods,0,,,,21,Photo,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,625,Cameras & Photo,15200,50508,50508,50508,50508,50508,Camera & Photo Accessories,LCD Hoods,LCD Hoods,LCD Hoods,LCD Hoods,LCD Hoods,262144,N,-999,Photo,6256,3170,50508,21,1,Electronics
-9426,Mobile Phones & Communication:Home Phones & Accessories:Phone Accessories:Other Phone Accessories,3,,,,31,Mobile & Home Phones,2011-04-05,2012-05-01 08:54:26,USER_X,USER_Y,15032,Mobile Phones & Communication,3286,9422,9426,9426,9426,9426,Home Phones & Accessories,Phone Accessories,Other Phone Accessories,Other Phone Accessories,Other Phone Accessories,Other Phone Accessories,262144,N,-999,Telecomm,-999,-999,9426,31,1,Electronics
-45238,Abbigliamento e accessori:Donna: Accessori:SciarpFoulard e Scialli,101,,,,9,Vestiti ed Accessori,2012-09-11,2012-09-11 20:27:22,USER_X,USER_Y,11450,Abbigliamento e accessori,4251,45238,45238,45238,45238,45238,Donna: Accessori,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,0,N,-999,Clothing & Accessories,-999,-999,45238,9,1,Fashion
-75665,Home & Garden:YarGarden & Outdoor Living:Gardening Supplies:Garden Tools & Equipment:Composting,0,,,,16,Home Improvement,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,159912,2032,29515,75665,75665,75665,YarGarden & Outdoor Living,Gardening Supplies,Garden Tools & Equipment,Composting,Composting,Composting,262144,N,-999,Home Improvement,-999,-999,75665,16,1,Home & Garden
-57784,ClothinShoes & Accessories:Baby & Toddler Clothing:Boys' Clothing (Newborn-5T):One-Pieces,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3082,147317,57784,57784,57784,57784,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),One-Pieces,One-Pieces,One-Pieces,One-Pieces,262144,N,-999,Clothing & Accessories,-999,-999,57784,9,1,Fashion
-6762,eBay Motors:Parts & Accessories:Manuals:Manuals:Other Makes,0,,6001,Other,-999,Unknown,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,-999,Unknown,-999,-999,-999,-999,-999,-999,Unknown,Unknown,Unknown,Unknown,Unknown,Unknown,0,N,-999,Unknown,-999,-999,-999,-999,1,Unknown
-60340,Entertainment Memorabilia:Movie Memorabilia:Pressbooks:1970-79,0,,,,42,Entertainment Memorabilia,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,45100,Entertainment Memorabilia,196,18830,60340,60340,60340,60340,Movie Memorabilia,Pressbooks,1970-79,1970-79,1970-79,1970-79,262144,N,-999,Entertainment Memorabilia,-999,-999,60340,42,1,Collectibles
-35570,eBay Motors:Parts & Accessories:Motorcycle Parts:DecalEmblems,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,10063,35570,35570,35570,35570,Parts & Accessories,Motorcycle Parts,DecalEmblems,DecalEmblems,DecalEmblems,DecalEmblems,262144,N,-999,Auto - Parts,5407,2940,35570,8,1,Parts & Accessories
-11554,ClothinShoes & Accessories:Women's Clothing:Jeans,0,,11450,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,11554,11554,11554,11554,11554,Women's Clothing,Jeans,Jeans,Jeans,Jeans,Jeans,262144,N,-999,Clothing & Accessories,7720,1113,11554,9,1,Fashion
-1357,Collectibles:Decorative Collectibles:Decorative Collectible Brands:Hamilton Collector Plates,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,156286,1357,1357,1357,1357,Decorative Collectibles,Decorative Collectible Brands,Hamilton Collector Plates,Hamilton Collector Plates,Hamilton Collector Plates,Hamilton Collector Plates,262144,N,-999,Collectibles,-999,-999,1357,11,1,Collectibles
-13987,Collectibles:Paper:Booklets:1920-39,0,,1,,11,Collectibles,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,1,Collectibles,124,13985,13987,13987,13987,13987,Paper,Booklets,1920-39,1920-39,1920-39,1920-39,262144,N,-999,Collectibles,-999,-999,13987,11,1,Collectibles
-63861,ClotheShoes & Accessories:Women's Clothing:Dresses,3,,,,9,Clothing & Accessories,2009-06-15,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,15724,63861,63861,63861,63861,63861,Women's Clothing,Dresses,Dresses,Dresses,Dresses,Dresses,262144,N,-999,Clothing & Accessories,-999,-999,63861,9,1,Fashion
-314,ClothinShoes & Accessories:Womens' Clothing:Other,211,,,,9,ClothinShoes & Accessories,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,314,314,314,314,314,Womens' Clothing,Other,Other,Other,Other,Other,0,N,-999,Clothing & Accessories,-999,-999,314,9,1,Fashion
-139973,Video Games & Consoles:Video Games,0,,,,32,Video Games,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Video Games,Video Games,Video Games,Video Games,Video Games,Video Games,262144,N,-999,Video Games,1453,1135,139973,32,1,Electronics
-30059,Cameras & Photography:Lenses & Filters:Lens AdapterMounts & Tubes,3,,,,21,Photography,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,625,Cameras & Photography,78997,30059,30059,30059,30059,30059,Lenses & Filters,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,262144,N,-999,Photo,4379,2630,30059,21,1,Electronics
-16145,Computers/Tablets & Networking:Computer Components & Parts:Other,0,,,,12,Computers,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,16145,16145,16145,16145,16145,Computer Components & Parts,Other,Other,Other,Other,Other,262144,N,-999,Computers,7312,3189,16145,12,1,Electronics
-95672,ClothinShoes & Accessories:Women's Shoes:Athletic,0,,,,9,Apparel,2011-04-05,2013-02-20 23:11:43,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,95672,95672,95672,95672,95672,Women's Shoes,Athletic,Athletic,Athletic,Athletic,Athletic,262144,N,-999,Clothing & Accessories,1251,1100,95672,9,1,Fashion
-1349,Collectibles:Decorative Collectibles:Decorative Collectible Brands:Avon:Bottles,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,156286,13779,1349,1349,1349,Decorative Collectibles,Decorative Collectible Brands,Avon,Bottles,Bottles,Bottles,262144,N,-999,Collectibles,-999,-999,1349,11,1,Collectibles
-1504,Business & Industrial:Electrical & Test Equipment:Test Equipment:Other,0,,9428,Other,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,92074,4676,1504,1504,1504,1504,Electrical & Test Equipment,Test Equipment,Other,Other,Other,Other,262144,N,-999,Business (Office & Industrial),-999,-999,1504,4,1,Business & Industrial
-73506,Collectibles:Decorative Collectibles:Tea PotSets,0,,,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,73506,73506,73506,73506,73506,Decorative Collectibles,Tea PotSets,Tea PotSets,Tea PotSets,Tea PotSets,Tea PotSets,262144,N,-999,Collectibles,-999,-999,73506,11,1,Collectibles
-38238,Home & Garden:Home Decor & Accents:Wall Decor:Other,0,,,,36,Home & Living,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,10033,10034,10034,10034,10034,10034,Home Decor,Other,Other,Other,Other,Other,262144,N,-999,Home Furnishing,2919,1868,10034,36,1,Home & Garden
-533,Coins & Paper Money:Coins: World:Africa:Other,0,,11116,Other,10,Coins,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,11116,Coins & Paper Money,256,127144,533,533,533,533,Coins: World,Africa,Other,Other,Other,Other,262144,N,2431,Coins,6213,1620,533,10,1,Collectibles
-223,Toys & Hobbies:Diecast & Toy Vehicles:CarTrucks & Vans:Diecast-Modern Manufacture,0,,220,Other,28,Toys,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,220,Toys & Hobbies,222,19036,223,223,223,223,Diecast & Toy Vehicles,CarTrucks & Vans,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,262144,N,-999,Toys,1740,1380,223,28,1,Collectibles
-31519,Computers/Tablets & Networking:Laptop & Desktop Accessories:Laptop Cases & Bags,3,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,31530,31519,31519,31519,31519,31519,Laptop & Desktop Accessories,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,262144,N,-999,Computers,6592,3189,31519,12,1,Electronics
-155226,ClothinShoes & Accessories:Women's Clothing:Sweats & Hoodies,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,155226,155226,155226,155226,155226,Women's Clothing,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,262144,N,-999,Clothing & Accessories,-999,-999,155226,9,1,Fashion
-63889,ClothinShoes & Accessories:Women's Shoes:Mixed Items & Lots,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,63889,63889,63889,63889,63889,Women's Shoes,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,262144,N,-999,Clothing & Accessories,-999,-999,63889,9,1,Fashion
-314,ClothinShoes & Accessories:Women's Clothing:Other,0,,11450,Other,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,314,314,314,314,314,Women's Clothing,Other,Other,Other,Other,Other,262144,N,-999,Clothing & Accessories,-999,-999,314,9,1,Fashion
[06/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
deleted file mode 100644
index 44bc2c3..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hdfs.web.JsonUtil;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.*;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by honma on 9/3/14.
- * <p/>
- * This tool serves for the purpose of migrating cubes. e.g. upgrade cube from
- * dev env to test(prod) env, or vice versa.
- * <p/>
- * Note that different envs are assumed to share the same hadoop cluster,
- * including hdfs, hbase and hive.
- */
-public class CubeMigrationCLI {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeMigrationCLI.class);
-
- private static List<Opt> operations;
- private static KylinConfig srcConfig;
- private static KylinConfig dstConfig;
- private static ResourceStore srcStore;
- private static ResourceStore dstStore;
- private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
-
- public static final String ACL_INFO_FAMILY = "i";
- private static final String ACL_TABLE_NAME = "_acl";
- private static final String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
- private static final String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
- private static final String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
-
- public static void main(String[] args) throws IOException, InterruptedException {
-
- if (args.length != 8) {
- usage();
- System.exit(1);
- }
-
- moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
- }
-
- private static void usage() {
- System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute");
- System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "purgeOrNot: true or false: whether purge the cube from src server after the migration. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
-
- }
-
- public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
-
- srcConfig = srcCfg;
- srcStore = ResourceStore.getStore(srcConfig);
- dstConfig = dstCfg;
- dstStore = ResourceStore.getStore(dstConfig);
-
- CubeManager cubeManager = CubeManager.getInstance(srcConfig);
- CubeInstance cube = cubeManager.getCube(cubeName);
- logger.info("cube to be moved is : " + cubeName);
-
- if (cube.getStatus() != RealizationStatusEnum.READY)
- throw new IllegalStateException("Cannot migrate cube that is not in READY state.");
-
- for (CubeSegment segment : cube.getSegments()) {
- if (segment.getStatus() != SegmentStatusEnum.READY) {
- throw new IllegalStateException("At least one segment is not in READY state");
- }
- }
-
- checkAndGetHbaseUrl();
-
- Configuration conf = HBaseConfiguration.create();
- hbaseAdmin = new HBaseAdmin(conf);
-
- hdfsFS = FileSystem.get(new Configuration());
-
- operations = new ArrayList<Opt>();
-
- copyFilesInMetaStore(cube, overwriteIfExists);
- renameFoldersInHdfs(cube);
- changeHtableHost(cube);
- addCubeIntoProject(cubeName, projectName);
- if (Boolean.parseBoolean(copyAcl) == true) {
- copyACL(cube, projectName);
- }
-
- if (Boolean.parseBoolean(purgeAndDisable) == true) {
- purgeAndDisable(cubeName); // this should be the last action
- }
-
- if (realExecute.equalsIgnoreCase("true")) {
- doOpts();
- } else {
- showOpts();
- }
- }
-
- public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
-
- moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
- }
-
- private static String checkAndGetHbaseUrl() {
- String srcMetadataUrl = srcConfig.getMetadataUrl();
- String dstMetadataUrl = dstConfig.getMetadataUrl();
-
- logger.info("src metadata url is " + srcMetadataUrl);
- logger.info("dst metadata url is " + dstMetadataUrl);
-
- int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase");
- int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase");
- if (srcIndex < 0 || dstIndex < 0)
- throw new IllegalStateException("Both metadata urls should be hbase metadata url");
-
- String srcHbaseUrl = srcMetadataUrl.substring(srcIndex).trim();
- String dstHbaseUrl = dstMetadataUrl.substring(dstIndex).trim();
- if (!srcHbaseUrl.equalsIgnoreCase(dstHbaseUrl)) {
- throw new IllegalStateException("hbase url not equal! ");
- }
-
- logger.info("hbase url is " + srcHbaseUrl.trim());
- return srcHbaseUrl.trim();
- }
-
- private static void renameFoldersInHdfs(CubeInstance cube) {
- for (CubeSegment segment : cube.getSegments()) {
-
- String jobUuid = segment.getLastBuildJobID();
- String src = JobInstance.getJobWorkingDir(jobUuid, srcConfig.getHdfsWorkingDirectory());
- String tgt = JobInstance.getJobWorkingDir(jobUuid, dstConfig.getHdfsWorkingDirectory());
-
- operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt }));
- }
-
- }
-
- private static void changeHtableHost(CubeInstance cube) {
- for (CubeSegment segment : cube.getSegments()) {
- operations.add(new Opt(OptType.CHANGE_HTABLE_HOST, new Object[] { segment.getStorageLocationIdentifier() }));
- }
- }
-
- private static void copyACL(CubeInstance cube, String projectName) {
- operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid(), cube.getDescriptor().getModel().getUuid(), projectName }));
- }
-
- private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
-
- List<String> metaItems = new ArrayList<String>();
- List<String> dictAndSnapshot = new ArrayList<String>();
- listCubeRelatedResources(cube, metaItems, dictAndSnapshot);
-
- if (dstStore.exists(cube.getResourcePath()) && !overwriteIfExists.equalsIgnoreCase("true"))
- throw new IllegalStateException("The cube named " + cube.getName() + " already exists on target metadata store. Use overwriteIfExists to overwrite it");
-
- for (String item : metaItems) {
- operations.add(new Opt(OptType.COPY_FILE_IN_META, new Object[] { item }));
- }
-
- for (String item : dictAndSnapshot) {
- operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
- }
- }
-
- private static void addCubeIntoProject(String cubeName, String projectName) throws IOException {
- String projectResPath = ProjectInstance.concatResourcePath(projectName);
- if (!dstStore.exists(projectResPath))
- throw new IllegalStateException("The target project " + projectName + "does not exist");
-
- operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { cubeName, projectName }));
- }
-
-
- private static void purgeAndDisable(String cubeName) throws IOException {
- operations.add(new Opt(OptType.PURGE_AND_DISABLE, new Object[] { cubeName }));
- }
-
- private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException {
-
- CubeDesc cubeDesc = cube.getDescriptor();
- metaResource.add(cube.getResourcePath());
- metaResource.add(cubeDesc.getResourcePath());
- metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName()));
-
- for (String table : cubeDesc.getModel().getAllTables()) {
- metaResource.add(TableDesc.concatResourcePath(table.toUpperCase()));
- }
-
- for (CubeSegment segment : cube.getSegments()) {
- dictAndSnapshot.addAll(segment.getSnapshotPaths());
- dictAndSnapshot.addAll(segment.getDictionaryPaths());
- }
- }
-
- private static enum OptType {
- COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL, PURGE_AND_DISABLE
- }
-
- private static class Opt {
- private OptType type;
- private Object[] params;
-
- private Opt(OptType type, Object[] params) {
- this.type = type;
- this.params = params;
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(type).append(":");
- for (Object s : params)
- sb.append(s).append(", ");
- return sb.toString();
- }
-
- }
-
- private static void showOpts() {
- for (int i = 0; i < operations.size(); ++i) {
- showOpt(operations.get(i));
- }
- }
-
- private static void showOpt(Opt opt) {
- logger.info("Operation: " + opt.toString());
- }
-
- private static void doOpts() throws IOException, InterruptedException {
- int index = 0;
- try {
- for (; index < operations.size(); ++index) {
- logger.info("Operation index :" + index);
- doOpt(operations.get(index));
- }
- } catch (Exception e) {
- logger.error("error met", e);
- logger.info("Try undoing previous changes");
- // undo:
- for (int i = index; i >= 0; --i) {
- try {
- undo(operations.get(i));
- } catch (Exception ee) {
- logger.error("error met ", e);
- logger.info("Continue undoing...");
- }
- }
-
- throw new RuntimeException("Cube moving failed");
- }
- }
-
- private static void doOpt(Opt opt) throws IOException, InterruptedException {
- logger.info("Executing operation: " + opt.toString());
-
- switch (opt.type) {
- case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
- desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
- logger.info("CHANGE_HTABLE_HOST is completed");
- break;
- }
- case COPY_FILE_IN_META: {
- String item = (String) opt.params[0];
- RawResource res = srcStore.getResource(item);
- dstStore.putResource(item, res.inputStream, res.timestamp);
- res.inputStream.close();
- logger.info("Item " + item + " is copied");
- break;
- }
- case COPY_DICT_OR_SNAPSHOT: {
- String item = (String) opt.params[0];
-
- if (item.toLowerCase().endsWith(".dict")) {
- DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig);
- DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig);
- DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
-
- long ts = dictSrc.getLastModified();
- dictSrc.setLastModified(0);//to avoid resource store write conflict
- DictionaryInfo dictSaved = dstDictMgr.trySaveNewDict(dictSrc.getDictionaryObject(), dictSrc);
- dictSrc.setLastModified(ts);
-
- if (dictSaved == dictSrc) {
- //no dup found, already saved to dest
- logger.info("Item " + item + " is copied");
- } else {
- //dictSrc is rejected because of duplication
- //modify cube's dictionary path
- String cubeName = (String) opt.params[1];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- for (CubeSegment segment : cube.getSegments()) {
- for (Map.Entry<String, String> entry : segment.getDictionaries().entrySet()) {
- if (entry.getValue().equalsIgnoreCase(item)) {
- entry.setValue(dictSaved.getResourcePath());
- }
- }
- }
- dstStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Item " + item + " is dup, instead " + dictSaved.getResourcePath() + " is reused");
- }
-
- } else if (item.toLowerCase().endsWith(".snapshot")) {
- SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig);
- SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig);
- SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
-
- long ts = snapSrc.getLastModified();
- snapSrc.setLastModified(0);
- SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc);
- snapSrc.setLastModified(ts);
-
- if (snapSaved == snapSrc) {
- //no dup found, already saved to dest
- logger.info("Item " + item + " is copied");
-
- } else {
- String cubeName = (String) opt.params[1];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = dstStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- for (CubeSegment segment : cube.getSegments()) {
- for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) {
- if (entry.getValue().equalsIgnoreCase(item)) {
- entry.setValue(snapSaved.getResourcePath());
- }
- }
- }
- dstStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");
-
- }
-
- } else {
- logger.error("unknown item found: " + item);
- logger.info("ignore it");
- }
-
- break;
- }
- case RENAME_FOLDER_IN_HDFS: {
- String srcPath = (String) opt.params[0];
- String dstPath = (String) opt.params[1];
- hdfsFS.rename(new Path(srcPath), new Path(dstPath));
- logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
- break;
- }
- case ADD_INTO_PROJECT: {
- String cubeName = (String) opt.params[0];
- String projectName = (String) opt.params[1];
- String projectResPath = ProjectInstance.concatResourcePath(projectName);
- Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
- ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
- project.removeRealization(RealizationType.CUBE, cubeName);
- project.addRealizationEntry(RealizationType.CUBE, cubeName);
- dstStore.putResource(projectResPath, project, projectSerializer);
- logger.info("Project instance for " + projectName + " is corrected");
- break;
- }
- case COPY_ACL: {
- String cubeId = (String) opt.params[0];
- String modelId = (String) opt.params[1];
- String projectName = (String) opt.params[2];
- String projectResPath = ProjectInstance.concatResourcePath(projectName);
- Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
- ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
- String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
- try {
- srcAclHtable = HBaseConnection.get(srcConfig.getMetadataUrl()).getTable(srcConfig.getMetadataUrlPrefix() + "_acl");
- destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
-
- // cube acl
- Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
- if (result.listCells() != null) {
- for (Cell cell : result.listCells()) {
- byte[] family = CellUtil.cloneFamily(cell);
- byte[] column = CellUtil.cloneQualifier(cell);
- byte[] value = CellUtil.cloneValue(cell);
-
- // use the target project uuid as the parent
- if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
- String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
- value = Bytes.toBytes(valueString);
- }
- Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
- destAclHtable.put(put);
- }
- }
- destAclHtable.flushCommits();
- } finally {
- IOUtils.closeQuietly(srcAclHtable);
- IOUtils.closeQuietly(destAclHtable);
- }
- break;
- }
- case PURGE_AND_DISABLE:{
- String cubeName = (String) opt.params[0];
- String cubeResPath = CubeInstance.concatResourcePath(cubeName);
- Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
- CubeInstance cube = srcStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
- cube.getSegments().clear();
- cube.setStatus(RealizationStatusEnum.DISABLED);
- srcStore.putResource(cubeResPath, cube, cubeSerializer);
- logger.info("Cube " + cubeName + " is purged and disabled in " + srcConfig.getMetadataUrl());
- }
- }
- }
-
- private static void undo(Opt opt) throws IOException, InterruptedException {
- logger.info("Undo operation: " + opt.toString());
-
- switch (opt.type) {
- case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
- desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
- break;
- }
- case COPY_FILE_IN_META: {
- // no harm
- logger.info("Undo for COPY_FILE_IN_META is ignored");
- break;
- }
- case COPY_DICT_OR_SNAPSHOT: {
- // no harm
- logger.info("Undo for COPY_DICT_OR_SNAPSHOT is ignored");
- break;
- }
- case RENAME_FOLDER_IN_HDFS: {
- String srcPath = (String) opt.params[1];
- String dstPath = (String) opt.params[0];
-
- if (hdfsFS.exists(new Path(srcPath)) && !hdfsFS.exists(new Path(dstPath))) {
- hdfsFS.rename(new Path(srcPath), new Path(dstPath));
- logger.info("HDFS Folder renamed from " + srcPath + " to " + dstPath);
- }
- break;
- }
- case ADD_INTO_PROJECT: {
- logger.info("Undo for ADD_INTO_PROJECT is ignored");
- break;
- }
- case COPY_ACL: {
- String cubeId = (String) opt.params[0];
- String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
- try {
- destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
-
- destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
- destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
- } finally {
- IOUtils.closeQuietly(destAclHtable);
- }
- break;
- }
- case PURGE_AND_DISABLE: {
- logger.info("Undo for PURGE_AND_DISABLE is not supported");
- break;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/DefaultSslProtocolSocketFactory.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/DefaultSslProtocolSocketFactory.java b/job/src/main/java/org/apache/kylin/job/tools/DefaultSslProtocolSocketFactory.java
deleted file mode 100644
index 3a2a88c..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/DefaultSslProtocolSocketFactory.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.UnknownHostException;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-
-import org.apache.commons.httpclient.ConnectTimeoutException;
-import org.apache.commons.httpclient.HttpClientError;
-import org.apache.commons.httpclient.params.HttpConnectionParams;
-import org.apache.commons.httpclient.protocol.ControllerThreadSocketFactory;
-import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class DefaultSslProtocolSocketFactory implements SecureProtocolSocketFactory {
- /** Log object for this class. */
- private static Logger LOG = LoggerFactory.getLogger(DefaultSslProtocolSocketFactory.class);
- private SSLContext sslcontext = null;
-
- /**
- * Constructor for DefaultSslProtocolSocketFactory.
- */
- public DefaultSslProtocolSocketFactory() {
- super();
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int)
- */
- public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
- }
-
- /**
- * Attempts to get a new socket connection to the given host within the
- * given time limit.
- *
- * <p>
- * To circumvent the limitations of older JREs that do not support connect
- * timeout a controller thread is executed. The controller thread attempts
- * to create a new socket within the given limit of time. If socket
- * constructor does not return until the timeout expires, the controller
- * terminates and throws an {@link ConnectTimeoutException}
- * </p>
- *
- * @param host
- * the host name/IP
- * @param port
- * the port on the host
- * @param localAddress
- * the local host name/IP to bind the socket to
- * @param localPort
- * the port on the local machine
- * @param params
- * {@link HttpConnectionParams Http connection parameters}
- *
- * @return Socket a new socket
- *
- * @throws IOException
- * if an I/O error occurs while creating the socket
- * @throws UnknownHostException
- * if the IP address of the host cannot be determined
- * @throws ConnectTimeoutException
- * DOCUMENT ME!
- * @throws IllegalArgumentException
- * DOCUMENT ME!
- */
- public Socket createSocket(final String host, final int port, final InetAddress localAddress, final int localPort, final HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
- if (params == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
-
- int timeout = params.getConnectionTimeout();
-
- if (timeout == 0) {
- return createSocket(host, port, localAddress, localPort);
- } else {
- // To be eventually deprecated when migrated to Java 1.4 or above
- return ControllerThreadSocketFactory.createSocket(this, host, port, localAddress, localPort, timeout);
- }
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int)
- */
- public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port);
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean)
- */
- public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(socket, host, port, autoClose);
- }
-
- public boolean equals(Object obj) {
- return ((obj != null) && obj.getClass().equals(DefaultX509TrustManager.class));
- }
-
- public int hashCode() {
- return DefaultX509TrustManager.class.hashCode();
- }
-
- private static SSLContext createEasySSLContext() {
- try {
- SSLContext context = SSLContext.getInstance("TLS");
- context.init(null, new TrustManager[] { new DefaultX509TrustManager(null) }, null);
-
- return context;
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new HttpClientError(e.toString());
- }
- }
-
- private SSLContext getSSLContext() {
- if (this.sslcontext == null) {
- this.sslcontext = createEasySSLContext();
- }
-
- return this.sslcontext;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/DefaultX509TrustManager.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/DefaultX509TrustManager.java b/job/src/main/java/org/apache/kylin/job/tools/DefaultX509TrustManager.java
deleted file mode 100644
index bd28245..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/DefaultX509TrustManager.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class DefaultX509TrustManager implements X509TrustManager {
-
- /** Log object for this class. */
- private static Logger LOG = LoggerFactory.getLogger(DefaultX509TrustManager.class);
- private X509TrustManager standardTrustManager = null;
-
- /**
- * Constructor for DefaultX509TrustManager.
- *
- */
- public DefaultX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
- super();
-
- TrustManagerFactory factory = TrustManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- factory.init(keystore);
-
- TrustManager[] trustmanagers = factory.getTrustManagers();
-
- if (trustmanagers.length == 0) {
- throw new NoSuchAlgorithmException("SunX509 trust manager not supported");
- }
-
- this.standardTrustManager = (X509TrustManager) trustmanagers[0];
- }
-
- public X509Certificate[] getAcceptedIssuers() {
- return this.standardTrustManager.getAcceptedIssuers();
- }
-
- public boolean isClientTrusted(X509Certificate[] certificates) {
- return true;
- // return this.standardTrustManager.isClientTrusted(certificates);
- }
-
- public boolean isServerTrusted(X509Certificate[] certificates) {
- if ((certificates != null) && LOG.isDebugEnabled()) {
- LOG.debug("Server certificate chain:");
-
- for (int i = 0; i < certificates.length; i++) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("X509Certificate[" + i + "]=" + certificates[i]);
- }
- }
- }
-
- if ((certificates != null) && (certificates.length == 1)) {
- X509Certificate certificate = certificates[0];
-
- try {
- certificate.checkValidity();
- } catch (CertificateException e) {
- LOG.error(e.toString());
-
- return false;
- }
-
- return true;
- } else {
- return true;
- // return this.standardTrustManager.isServerTrusted(certificates);
- }
- }
-
- @Override
- public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- // TODO Auto-generated method stub
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java b/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
deleted file mode 100644
index bf655a0..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
+++ /dev/null
@@ -1,384 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class DeployCoprocessorCLI {
-
- public static final String CubeObserverClassV2 = "org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.AggregateRegionObserver";
- public static final String CubeEndpointClassV2 = "org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService";
- public static final String IIEndpointClassV2 = "org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint";
- public static final String OBSERVER_CLS_NAME = "org.apache.kylin.storage.hbase.coprocessor.observer.AggregateRegionObserver";
- public static final String ENDPOINT_CLS_NAMAE = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
- private static final Logger logger = LoggerFactory.getLogger(DeployCoprocessorCLI.class);
-
- public static void main(String[] args) throws IOException {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
-
- String localCoprocessorJar = new File(args[0]).getAbsolutePath();
- logger.info("Identify coprocessor jar " + localCoprocessorJar);
-
- List<String> tableNames = getHTableNames(kylinConfig);
- logger.info("Identify tables " + tableNames);
-
- if (args.length <= 1) {
- printUsageAndExit();
- }
-
- String filterType = args[1].toLowerCase();
- if (filterType.equals("-table")) {
- tableNames = filterByTables(tableNames, Arrays.asList(args).subList(2, args.length));
- } else if (filterType.equals("-cube")) {
- tableNames = filterByCubes(tableNames, Arrays.asList(args).subList(2, args.length));
- } else if (!filterType.equals("all")) {
- printUsageAndExit();
- }
-
- logger.info("Will execute tables " + tableNames);
-
- Set<String> oldJarPaths = getCoprocessorJarPaths(hbaseAdmin, tableNames);
- logger.info("Old coprocessor jar: " + oldJarPaths);
-
- Path hdfsCoprocessorJar = uploadCoprocessorJar(localCoprocessorJar, fileSystem, oldJarPaths);
- logger.info("New coprocessor jar: " + hdfsCoprocessorJar);
-
- List<String> processedTables = resetCoprocessorOnHTables(hbaseAdmin, hdfsCoprocessorJar, tableNames);
-
- // Don't remove old jars, missing coprocessor jar will fail hbase
- // removeOldJars(oldJarPaths, fileSystem);
-
- hbaseAdmin.close();
-
- logger.info("Processed " + processedTables);
- logger.info("Active coprocessor jar: " + hdfsCoprocessorJar);
- }
-
- private static void printUsageAndExit() {
- logger.warn("Probe run, exiting.");
- logger.info("Usage: bin/kylin.sh org.apache.kylin.job.tools.DeployCoprocessorCLI JAR_FILE all|-cube CUBE1 CUBE2|-table TABLE1 TABLE2");
- System.exit(0);
- }
-
- private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<String> result = Lists.newArrayList();
- for (String c : cubeNames) {
- c = c.trim();
- if (c.endsWith(","))
- c = c.substring(0, c.length() - 1);
-
- CubeInstance cubeInstance = cubeManager.getCube(c);
- for (CubeSegment segment : cubeInstance.getSegments()) {
- String tableName = segment.getStorageLocationIdentifier();
- if (allTableNames.contains(tableName)) {
- result.add(tableName);
- }
- }
- }
- return result;
- }
-
- private static List<String> filterByTables(List<String> allTableNames, List<String> tableNames) {
- List<String> result = Lists.newArrayList();
- for (String t : tableNames) {
- t = t.trim();
- if (t.endsWith(","))
- t = t.substring(0, t.length() - 1);
-
- if (allTableNames.contains(t)) {
- result.add(t);
- }
- }
- return result;
- }
-
- public static void deployCoprocessor(HTableDescriptor tableDesc) {
- try {
- initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
-
- } catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
- logger.error("Will try creating the table without coprocessor.");
- }
- }
-
- private static void initHTableCoprocessor(HTableDescriptor desc) throws IOException {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- String localCoprocessorJar = kylinConfig.getCoprocessorLocalJar();
- Path hdfsCoprocessorJar = DeployCoprocessorCLI.uploadCoprocessorJar(localCoprocessorJar, fileSystem, null);
-
- DeployCoprocessorCLI.addCoprocessorOnHTable(desc, hdfsCoprocessorJar);
- }
-
- public static void addCoprocessorOnHTable(HTableDescriptor desc, Path hdfsCoprocessorJar) throws IOException {
- logger.info("Add coprocessor on " + desc.getNameAsString());
- desc.addCoprocessor(ENDPOINT_CLS_NAMAE, hdfsCoprocessorJar, 1000, null);
- desc.addCoprocessor(OBSERVER_CLS_NAME, hdfsCoprocessorJar, 1001, null);
- }
-
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
- logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
-
- logger.info("Unset coprocessor on " + tableName);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
-
- // remove coprocessors of 1.x version
- while (desc.hasCoprocessor(OBSERVER_CLS_NAME)) {
- desc.removeCoprocessor(OBSERVER_CLS_NAME);
- }
- while (desc.hasCoprocessor(ENDPOINT_CLS_NAMAE)) {
- desc.removeCoprocessor(ENDPOINT_CLS_NAMAE);
- }
- // remove coprocessors of 2.x version
- while (desc.hasCoprocessor(CubeObserverClassV2)) {
- desc.removeCoprocessor(CubeObserverClassV2);
- }
- while (desc.hasCoprocessor(CubeEndpointClassV2)) {
- desc.removeCoprocessor(CubeEndpointClassV2);
- }
- while (desc.hasCoprocessor(IIEndpointClassV2)) {
- desc.removeCoprocessor(IIEndpointClassV2);
- }
-
- addCoprocessorOnHTable(desc, hdfsCoprocessorJar);
- hbaseAdmin.modifyTable(tableName, desc);
-
- logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
- }
-
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
- List<String> processed = new ArrayList<String>();
-
- for (String tableName : tableNames) {
- try {
- resetCoprocessor(tableName, hbaseAdmin, hdfsCoprocessorJar);
- processed.add(tableName);
- } catch (IOException ex) {
- logger.error("Error processing " + tableName, ex);
- }
- }
- return processed;
- }
-
- public static Path getNewestCoprocessorJar(KylinConfig config, FileSystem fileSystem) throws IOException {
- Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, config);
- FileStatus newestJar = null;
- for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) {
- if (fileStatus.getPath().toString().endsWith(".jar")) {
- if (newestJar == null) {
- newestJar = fileStatus;
- } else {
- if (newestJar.getModificationTime() < fileStatus.getModificationTime())
- newestJar = fileStatus;
- }
- }
- }
- if (newestJar == null)
- return null;
-
- Path path = newestJar.getPath().makeQualified(fileSystem.getUri(), null);
- logger.info("The newest coprocessor is " + path.toString());
- return path;
- }
-
- public static Path uploadCoprocessorJar(String localCoprocessorJar, FileSystem fileSystem, Set<String> oldJarPaths) throws IOException {
- Path uploadPath = null;
- File localCoprocessorFile = new File(localCoprocessorJar);
-
- // check existing jars
- if (oldJarPaths == null) {
- oldJarPaths = new HashSet<String>();
- }
- Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, KylinConfig.getInstanceFromEnv());
- for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) {
- if (fileStatus.getLen() == localCoprocessorJar.length() && fileStatus.getModificationTime() == localCoprocessorFile.lastModified()) {
- uploadPath = fileStatus.getPath();
- break;
- }
- String filename = fileStatus.getPath().toString();
- if (filename.endsWith(".jar")) {
- oldJarPaths.add(filename);
- }
- }
-
- // upload if not existing
- if (uploadPath == null) {
- // figure out a unique new jar file name
- Set<String> oldJarNames = new HashSet<String>();
- for (String path : oldJarPaths) {
- oldJarNames.add(new Path(path).getName());
- }
- String baseName = getBaseFileName(localCoprocessorJar);
- String newName = null;
- int i = 0;
- while (newName == null) {
- newName = baseName + "-" + (i++) + ".jar";
- if (oldJarNames.contains(newName))
- newName = null;
- }
-
- // upload
- uploadPath = new Path(coprocessorDir, newName);
- FileInputStream in = null;
- FSDataOutputStream out = null;
- try {
- in = new FileInputStream(localCoprocessorFile);
- out = fileSystem.create(uploadPath);
- IOUtils.copy(in, out);
- } finally {
- IOUtils.closeQuietly(in);
- IOUtils.closeQuietly(out);
- }
-
- fileSystem.setTimes(uploadPath, localCoprocessorFile.lastModified(), -1);
-
- }
-
- uploadPath = uploadPath.makeQualified(fileSystem.getUri(), null);
- return uploadPath;
- }
-
- private static String getBaseFileName(String localCoprocessorJar) {
- File localJar = new File(localCoprocessorJar);
- String baseName = localJar.getName();
- if (baseName.endsWith(".jar"))
- baseName = baseName.substring(0, baseName.length() - ".jar".length());
- return baseName;
- }
-
- private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
- String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
- Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
- fileSystem.mkdirs(coprocessorDir);
- return coprocessorDir;
- }
-
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
- HashSet<String> result = new HashSet<String>();
-
- for (String tableName : tableNames) {
- HTableDescriptor tableDescriptor = null;
- try {
- tableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- } catch (TableNotFoundException e) {
- logger.warn("Table not found " + tableName, e);
- continue;
- }
-
- Matcher keyMatcher;
- Matcher valueMatcher;
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e : tableDescriptor.getValues().entrySet()) {
- keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e.getKey().get()));
- if (!keyMatcher.matches()) {
- continue;
- }
- valueMatcher = HConstants.CP_HTD_ATTR_VALUE_PATTERN.matcher(Bytes.toString(e.getValue().get()));
- if (!valueMatcher.matches()) {
- continue;
- }
-
- String jarPath = valueMatcher.group(1).trim();
- String clsName = valueMatcher.group(2).trim();
-
- if (OBSERVER_CLS_NAME.equals(clsName)) {
- result.add(jarPath);
- }
- }
- }
-
- return result;
- }
-
- private static List<String> getHTableNames(KylinConfig config) {
- CubeManager cubeMgr = CubeManager.getInstance(config);
-
- ArrayList<String> result = new ArrayList<String>();
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
- for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
- String tableName = seg.getStorageLocationIdentifier();
- if (StringUtils.isBlank(tableName) == false) {
- result.add(tableName);
- System.out.println("added new table: " + tableName);
- }
- }
- }
-
- for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) {
- for (IISegment seg : ii.getSegments(SegmentStatusEnum.READY)) {
- String tableName = seg.getStorageLocationIdentifier();
- if (StringUtils.isBlank(tableName) == false) {
- result.add(tableName);
- System.out.println("added new table: " + tableName);
- }
- }
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java b/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
deleted file mode 100644
index 70e1df6..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Pair;
-
-import com.google.common.collect.Lists;
-
-public class GridTableHBaseBenchmark {
-
- private static final String TEST_TABLE = "GridTableTest";
- private static final byte[] CF = "F".getBytes();
- private static final byte[] QN = "C".getBytes();
- private static final int N_ROWS = 10000;
- private static final int CELL_SIZE = 128 * 1024; // 128 KB
- private static final double DFT_HIT_RATIO = 0.3;
- private static final double DFT_INDEX_RATIO = 0.1;
- private static final int ROUND = 3;
-
- public static void main(String[] args) throws IOException {
- double hitRatio = DFT_HIT_RATIO;
- try {
- hitRatio = Double.parseDouble(args[0]);
- } catch (Exception e) {
- // nevermind
- }
-
- double indexRatio = DFT_INDEX_RATIO;
- try {
- indexRatio = Double.parseDouble(args[1]);
- } catch (Exception e) {
- // nevermind
- }
-
- testGridTable(hitRatio, indexRatio);
- }
-
- public static void testGridTable(double hitRatio, double indexRatio) throws IOException {
- System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
- String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
-
- HConnection conn = HBaseConnection.get(hbaseUrl);
- createHTableIfNeeded(conn, TEST_TABLE);
- prepareData(conn);
-
- Hits hits = new Hits(N_ROWS, hitRatio, indexRatio);
-
- for (int i = 0; i < ROUND; i++) {
- System.out.println("==================================== ROUND " + (i + 1) + " ========================================");
- testRowScanWithIndex(conn, hits.getHitsForRowScanWithIndex());
- testRowScanNoIndexFullScan(conn, hits.getHitsForRowScanNoIndex());
- testRowScanNoIndexSkipScan(conn, hits.getHitsForRowScanNoIndex());
- testColumnScan(conn, hits.getHitsForColumnScan());
- }
-
- }
-
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
- Stats stats = new Stats("COLUMN_SCAN");
-
- HTableInterface table = conn.getTable(TEST_TABLE);
- try {
- stats.markStart();
-
- int nLogicCols = colScans.size();
- int nLogicRows = colScans.get(0).getSecond() - colScans.get(0).getFirst();
-
- Scan[] scans = new Scan[nLogicCols];
- ResultScanner[] scanners = new ResultScanner[nLogicCols];
- for (int i = 0; i < nLogicCols; i++) {
- scans[i] = new Scan();
- scans[i].addFamily(CF);
- scanners[i] = table.getScanner(scans[i]);
- }
- for (int i = 0; i < nLogicRows; i++) {
- for (int c = 0; c < nLogicCols; c++) {
- Result r = scanners[c].next();
- stats.consume(r);
- }
- dot(i, nLogicRows);
- }
-
- stats.markEnd();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
- fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
- }
-
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
- jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
- }
-
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
- jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
- }
-
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
- try {
- stats.markStart();
-
- Scan scan = new Scan();
- scan.addFamily(CF);
- ResultScanner scanner = table.getScanner(scan);
- int i = 0;
- for (Result r : scanner) {
- if (hits[i])
- stats.consume(r);
- dot(i, N_ROWS);
- i++;
- }
-
- stats.markEnd();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
-
- final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
-
- HTableInterface table = conn.getTable(TEST_TABLE);
- try {
-
- stats.markStart();
-
- int i = 0;
- while (i < N_ROWS) {
- int start, end;
- for (start = i; start < N_ROWS; start++) {
- if (hits[start])
- break;
- }
- for (end = start + 1; end < N_ROWS; end++) {
- boolean isEnd = true;
- for (int j = 0; j < jumpThreshold && end + j < N_ROWS; j++)
- if (hits[end + j])
- isEnd = false;
- if (isEnd)
- break;
- }
-
- if (start < N_ROWS) {
- Scan scan = new Scan();
- scan.setStartRow(Bytes.toBytes(start));
- scan.setStopRow(Bytes.toBytes(end));
- scan.addFamily(CF);
- ResultScanner scanner = table.getScanner(scan);
- i = start;
- for (Result r : scanner) {
- stats.consume(r);
- dot(i, N_ROWS);
- i++;
- }
- }
- i = end;
- }
-
- stats.markEnd();
-
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
-
- try {
- // check how many rows existing
- int nRows = 0;
- Scan scan = new Scan();
- scan.setFilter(new KeyOnlyFilter());
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- r.getRow(); // nothing to do
- nRows++;
- }
-
- if (nRows > 0) {
- System.out.println(nRows + " existing rows");
- if (nRows != N_ROWS)
- throw new IOException("Expect " + N_ROWS + " rows but it is not");
- return;
- }
-
- // insert rows into empty table
- System.out.println("Writing " + N_ROWS + " rows to " + TEST_TABLE);
- long nBytes = 0;
- for (int i = 0; i < N_ROWS; i++) {
- byte[] rowkey = Bytes.toBytes(i);
- Put put = new Put(rowkey);
- byte[] cell = randomBytes();
- put.add(CF, QN, cell);
- table.put(put);
- nBytes += cell.length;
- dot(i, N_ROWS);
- }
- System.out.println();
- System.out.println("Written " + N_ROWS + " rows, " + nBytes + " bytes");
-
- } finally {
- IOUtils.closeQuietly(table);
- }
-
- }
-
- private static void dot(int i, int nRows) {
- if (i % (nRows / 100) == 0)
- System.out.print(".");
- }
-
- private static byte[] randomBytes() {
- byte[] bytes = new byte[CELL_SIZE];
- Random rand = new Random();
- rand.nextBytes(bytes);
- return bytes;
- }
-
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
- try {
- boolean tableExist = false;
- try {
- hbase.getTableDescriptor(TableName.valueOf(tableName));
- tableExist = true;
- } catch (TableNotFoundException e) {
- }
-
- if (tableExist) {
- System.out.println("HTable '" + tableName + "' already exists");
- return;
- }
-
- System.out.println("Creating HTable '" + tableName + "'");
-
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
-
- HColumnDescriptor fd = new HColumnDescriptor(CF);
- fd.setBlocksize(CELL_SIZE);
- desc.addFamily(fd);
- hbase.createTable(desc);
-
- System.out.println("HTable '" + tableName + "' created");
- } finally {
- hbase.close();
- }
- }
-
- static class Hits {
-
- boolean[] hitsForRowScanWithIndex;
- boolean[] hitsForRowScanNoIndex;
- List<Pair<Integer, Integer>> hitsForColumnScan;
-
- public Hits(int nRows, double hitRatio, double indexRatio) {
- Random rand = new Random();
-
- hitsForRowScanWithIndex = new boolean[nRows];
- hitsForRowScanNoIndex = new boolean[nRows];
-
- // for row scan
- int blockSize = (int) (1.0 / indexRatio);
- int nBlocks = nRows / blockSize;
-
- for (int i = 0; i < nBlocks; i++) {
-
- if (rand.nextDouble() < hitRatio) {
- for (int j = 0; j < blockSize; j++) {
- hitsForRowScanNoIndex[i * blockSize + j] = true;
- hitsForRowScanWithIndex[i * blockSize + j] = true;
- }
- } else {
- // case of not hit
- hitsForRowScanNoIndex[i * blockSize] = true;
- }
- }
-
- hitsForColumnScan = Lists.newArrayList();
-
- // for column scan
- int nColumns = 20;
- int logicRows = nRows / nColumns;
- for (int i = 0; i < nColumns; i++) {
- if (rand.nextDouble() < hitRatio) {
- hitsForColumnScan.add(new Pair<Integer, Integer>(i * logicRows, (i + 1) * logicRows));
- }
- }
-
- }
-
- public boolean[] getHitsForRowScanWithIndex() {
- return hitsForRowScanWithIndex;
- }
-
- public boolean[] getHitsForRowScanNoIndex() {
- return hitsForRowScanNoIndex;
- }
-
- public List<Pair<Integer, Integer>> getHitsForColumnScan() {
- return hitsForColumnScan;
- }
- }
-
- static class Stats {
- String name;
- long startTime;
- long endTime;
- long rowsRead;
- long bytesRead;
-
- public Stats(String name) {
- this.name = name;
- }
-
- public void consume(Result r) {
- consume(r, Integer.MAX_VALUE);
- }
-
- private void consume(Result r, int nBytesToConsume) {
- Cell cell = r.getColumnLatestCell(CF, QN);
- byte mix = 0;
- byte[] valueArray = cell.getValueArray();
- int n = Math.min(nBytesToConsume, cell.getValueLength());
- for (int i = 0; i < n; i++) {
- mix ^= valueArray[i];
- bytesRead++;
- }
- discard(mix);
- rowsRead++;
- }
-
- private void discard(byte n) {
- // do nothing
- }
-
- public void markStart() {
- System.out.println(name + " starts");
- startTime = System.currentTimeMillis();
- }
-
- public void markEnd() {
- endTime = System.currentTimeMillis();
- System.out.println();
- System.out.println(name + " ends, " + (endTime - startTime) + " ms, " + rowsRead + " rows read, " + bytesRead + " bytes read");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
deleted file mode 100644
index 6d741aa..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class HadoopStatusChecker {
-
- protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusChecker.class);
-
- private final String yarnUrl;
- private final String mrJobID;
- private final StringBuilder output;
- private final KylinConfig config;
-
- public HadoopStatusChecker(String yarnUrl, String mrJobID, StringBuilder output, KylinConfig config) {
- this.yarnUrl = yarnUrl;
- this.mrJobID = mrJobID;
- this.output = output;
- this.config = config;
- }
-
- public JobStepStatusEnum checkStatus() {
- if (null == mrJobID) {
- this.output.append("Skip status check with empty job id..\n");
- return JobStepStatusEnum.WAITING;
- }
- JobStepStatusEnum status = null;
- try {
- boolean useKerberosAuth = config.getKylinUseKerberosAuth();
- final Pair<RMAppState, FinalApplicationStatus> result = new HadoopStatusGetter(yarnUrl, mrJobID).get(useKerberosAuth);
- logger.debug("State of Hadoop job: " + mrJobID + ":" + result.getLeft() + "-" + result.getRight());
- output.append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").format(new Date()) + " - State of Hadoop job: " + mrJobID + ":" + result.getLeft() + " - " + result.getRight() + "\n");
-
- switch (result.getRight()) {
- case SUCCEEDED:
- status = JobStepStatusEnum.FINISHED;
- break;
- case FAILED:
- status = JobStepStatusEnum.ERROR;
- break;
- case KILLED:
- status = JobStepStatusEnum.KILLED;
- break;
- case UNDEFINED:
- switch (result.getLeft()) {
- case NEW:
- case NEW_SAVING:
- case SUBMITTED:
- case ACCEPTED:
- status = JobStepStatusEnum.WAITING;
- break;
- case RUNNING:
- status = JobStepStatusEnum.RUNNING;
- break;
- case FINAL_SAVING:
- case FINISHING:
- case FINISHED:
- case FAILED:
- case KILLING:
- case KILLED:
- }
- break;
- }
- } catch (Exception e) {
- logger.error("error check status", e);
- output.append("Exception: " + e.getLocalizedMessage() + "\n");
- status = JobStepStatusEnum.ERROR;
- }
-
- return status;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
deleted file mode 100644
index 9035ad4..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.security.Principal;
-
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.protocol.Protocol;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.http.auth.AuthSchemeProvider;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.client.config.AuthSchemes;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.config.Lookup;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by qianzhou on 1/20/15.
- */
-public class HadoopStatusGetter {
-
- private final String mrJobId;
- private final String yarnUrl;
-
- protected static final Logger log = LoggerFactory.getLogger(HadoopStatusChecker.class);
-
- public HadoopStatusGetter(String yarnUrl, String mrJobId) {
- this.yarnUrl = yarnUrl;
- this.mrJobId = mrJobId;
- }
-
- public Pair<RMAppState, FinalApplicationStatus> get(boolean useKerberos) throws IOException {
- String applicationId = mrJobId.replace("job", "application");
- String url = yarnUrl.replace("${job_id}", applicationId);
- String response = useKerberos ? getHttpResponseWithKerberosAuth(url) : getHttpResponse(url);
- JsonNode root = new ObjectMapper().readTree(response);
- RMAppState state = RMAppState.valueOf(root.findValue("state").getTextValue());
- FinalApplicationStatus finalStatus = FinalApplicationStatus.valueOf(root.findValue("finalStatus").getTextValue());
- return Pair.of(state, finalStatus);
- }
-
- private String getHttpResponse(String url) throws IOException {
- HttpClient client = new HttpClient();
-
- String response = null;
- while (response == null) { // follow redirects via 'refresh'
- if (url.startsWith("https://")) {
- registerEasyHttps();
- }
- if (url.contains("anonymous=true") == false) {
- url += url.contains("?") ? "&" : "?";
- url += "anonymous=true";
- }
-
- HttpMethod get = new GetMethod(url);
- get.addRequestHeader("accept", "application/json");
-
- try {
- client.executeMethod(get);
-
- String redirect = null;
- Header h = get.getResponseHeader("Location");
- if (h != null) {
- redirect = h.getValue();
- if (isValidURL(redirect) == false) {
- log.info("Get invalid redirect url, skip it: " + redirect);
- Thread.sleep(1000l);
- continue;
- }
- } else {
- h = get.getResponseHeader("Refresh");
- if (h != null) {
- String s = h.getValue();
- int cut = s.indexOf("url=");
- if (cut >= 0) {
- redirect = s.substring(cut + 4);
-
- if (isValidURL(redirect) == false) {
- log.info("Get invalid redirect url, skip it: " + redirect);
- Thread.sleep(1000l);
- continue;
- }
- }
- }
- }
-
- if (redirect == null) {
- response = get.getResponseBodyAsString();
- log.debug("Job " + mrJobId + " get status check result.\n");
- } else {
- url = redirect;
- log.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
- }
- } catch (InterruptedException e) {
- log.error(e.getMessage());
- } finally {
- get.releaseConnection();
- }
- }
-
- return response;
- }
-
- private static String DEFAULT_KRB5_CONFIG_LOCATION = "/etc/krb5.conf";
- private String getHttpResponseWithKerberosAuth(String url) throws IOException {
-
- // referred from https://stackoverflow.com/questions/24633380/how-do-i-authenticate-with-spnego-kerberos-and-apaches-httpclient
- String krb5ConfigPath = System.getProperty("java.security.krb5.conf");
- if (krb5ConfigPath == null) {
- krb5ConfigPath = DEFAULT_KRB5_CONFIG_LOCATION;
- }
- log.debug("krb5 config file is " + krb5ConfigPath);
-
- boolean skipPortAtKerberosDatabaseLookup = true;
- System.setProperty("java.security.krb5.conf", krb5ConfigPath);
- System.setProperty("sun.security.krb5.debug", "true");
- System.setProperty("javax.security.auth.useSubjectCredsOnly","false");
- Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup))
- .build();
-
- CloseableHttpClient client = HttpClients.custom().setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
- HttpClientContext context = HttpClientContext.create();
- BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
- // This may seem odd, but specifying 'null' as principal tells java to use the logged in user's credentials
- Credentials useJaasCreds = new Credentials() {
- public String getPassword() {
- return null;
- }
- public Principal getUserPrincipal() {
- return null;
- }
- };
- credentialsProvider.setCredentials( new AuthScope(null, -1, null), useJaasCreds );
- context.setCredentialsProvider(credentialsProvider);
- String responseString = null;
- int count = 0;
- int MAX_RETRY_TIME = 3;
- while(responseString == null && count ++ < MAX_RETRY_TIME) {
- if (url.startsWith("https://")) {
- registerEasyHttps();
- }
- if (url.contains("anonymous=true") == false) {
- url += url.contains("?") ? "&" : "?";
- url += "anonymous=true";
- }
- HttpGet httpget = new HttpGet(url);
- try {
- httpget.addHeader("accept", "application/json");
- CloseableHttpResponse response = client.execute(httpget,context);
- String redirect = null;
- org.apache.http.Header h = response.getFirstHeader("Location");
- if (h != null) {
- redirect = h.getValue();
- if (isValidURL(redirect) == false) {
- log.info("Get invalid redirect url, skip it: " + redirect);
- Thread.sleep(1000l);
- continue;
- }
- } else {
- h = response.getFirstHeader("Refresh");
- if (h != null) {
- String s = h.getValue();
- int cut = s.indexOf("url=");
- if (cut >= 0) {
- redirect = s.substring(cut + 4);
-
- if (isValidURL(redirect) == false) {
- log.info("Get invalid redirect url, skip it: " + redirect);
- Thread.sleep(1000l);
- continue;
- }
- }
- }
- }
-
- if (redirect == null) {
- responseString = IOUtils.toString(response.getEntity().getContent());
- log.debug("Job " + mrJobId + " get status check result.\n");
- } else {
- url = redirect;
- log.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
- }
- } catch (InterruptedException e) {
- log.error(e.getMessage());
- } finally {
- httpget.releaseConnection();
- }
- }
-
- return responseString;
- }
-
- private static Protocol EASY_HTTPS = null;
-
- private static void registerEasyHttps() {
- // by pass all https issue
- if (EASY_HTTPS == null) {
- EASY_HTTPS = new Protocol("https", (ProtocolSocketFactory) new DefaultSslProtocolSocketFactory(), 443);
- Protocol.registerProtocol("https", EASY_HTTPS);
- }
- }
-
- private static boolean isValidURL(String value) {
- if (StringUtils.isNotEmpty(value)) {
- java.net.URL url;
- try {
- url = new java.net.URL(value);
- } catch (MalformedURLException var5) {
- return false;
- }
-
- return StringUtils.isNotEmpty(url.getProtocol()) && StringUtils.isNotEmpty(url.getHost());
- }
-
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java b/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
deleted file mode 100644
index 53930e3..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/HtableAlterMetadataCLI.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by honma on 11/11/14.
- */
-@SuppressWarnings("static-access")
-public class HtableAlterMetadataCLI extends AbstractHadoopJob {
-
- private static final Option OPTION_METADATA_KEY = OptionBuilder.withArgName("key").hasArg().isRequired(true).withDescription("The metadata key").create("key");
- private static final Option OPTION_METADATA_VALUE = OptionBuilder.withArgName("value").hasArg().isRequired(true).withDescription("The metadata value").create("value");
-
- protected static final Logger log = LoggerFactory.getLogger(HtableAlterMetadataCLI.class);
-
- String tableName;
- String metadataKey;
- String metadataValue;
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
- try {
- options.addOption(OPTION_HTABLE_NAME);
- options.addOption(OPTION_METADATA_KEY);
- options.addOption(OPTION_METADATA_VALUE);
-
- parseOptions(options, args);
- tableName = getOptionValue(OPTION_HTABLE_NAME);
- metadataKey = getOptionValue(OPTION_METADATA_KEY);
- metadataValue = getOptionValue(OPTION_METADATA_VALUE);
-
- alter();
-
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- private void alter() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
- HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
-
- hbaseAdmin.disableTable(table.getTableName());
- table.setValue(metadataKey, metadataValue);
- hbaseAdmin.modifyTable(table.getTableName(), table);
- hbaseAdmin.enableTable(table.getTableName());
- hbaseAdmin.close();
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new HtableAlterMetadataCLI(), args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/OptionsHelper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/OptionsHelper.java b/job/src/main/java/org/apache/kylin/job/tools/OptionsHelper.java
deleted file mode 100644
index 5ed5b35..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/OptionsHelper.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class OptionsHelper {
- private CommandLine commandLine;
-
- public void parseOptions(Options options, String[] args) throws ParseException {
- CommandLineParser parser = new GnuParser();
- commandLine = parser.parse(options, args);
- }
-
- public Option[] getOptions() {
- return commandLine.getOptions();
- }
-
- public String getOptionsAsString() {
- StringBuilder buf = new StringBuilder();
- for (Option option : commandLine.getOptions()) {
- buf.append(" ");
- buf.append(option.getOpt());
- if (option.hasArg()) {
- buf.append("=");
- buf.append(option.getValue());
- }
- }
- return buf.toString();
- }
-
- public String getOptionValue(Option option) {
- return commandLine.getOptionValue(option.getOpt());
- }
-
- public boolean hasOption(Option option) {
- return commandLine.hasOption(option.getOpt());
- }
-
- public void printUsage(String programName, Options options) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(programName, options);
- }
-
- public static String convertToFileURL(String path) {
- if (File.separatorChar != '/') {
- path = path.replace(File.separatorChar, '/');
- }
-
- return path;
- }
-
-}
[45/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
deleted file mode 100644
index 646cd80..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-
-import com.google.common.collect.Lists;
-
-public class FileResourceStore extends ResourceStore {
-
- File root;
-
- public FileResourceStore(KylinConfig kylinConfig) {
- super(kylinConfig);
- root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile();
- if (root.exists() == false)
- throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
- }
-
- @Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
- String[] names = file(resPath).list();
- if (names == null) // not a directory
- return null;
-
- ArrayList<String> r = new ArrayList<String>(names.length);
- String prefix = resPath.endsWith("/") ? resPath : resPath + "/";
- for (String n : names) {
- r.add(prefix + n);
- }
- return r;
- }
-
- @Override
- protected boolean existsImpl(String resPath) throws IOException {
- File f = file(resPath);
- return f.exists() && f.isFile(); // directory is not considered a
- // resource
- }
-
- @Override
- protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException {
- List<RawResource> result = Lists.newArrayList();
- try {
- String commonPrefix = StringUtils.getCommonPrefix(rangeEnd, rangeStart);
- commonPrefix = commonPrefix.substring(0, commonPrefix.lastIndexOf("/") + 1);
- final ArrayList<String> resources = listResourcesImpl(commonPrefix);
- for (String resource : resources) {
- if (resource.compareTo(rangeStart) >= 0 && resource.compareTo(rangeEnd) <= 0) {
- if (existsImpl(resource)) {
- result.add(getResourceImpl(resource));
- }
- }
- }
- return result;
- } catch (IOException ex) {
- for (RawResource rawResource : result) {
- IOUtils.closeQuietly(rawResource.inputStream);
- }
- throw ex;
- } catch (Exception ex) {
- throw new UnsupportedOperationException(ex);
- }
- }
-
- @Override
- protected RawResource getResourceImpl(String resPath) throws IOException {
- File f = file(resPath);
- if (f.exists() && f.isFile())
- return new RawResource(new FileInputStream(f), f.lastModified());
- else
- return null;
- }
-
- @Override
- protected long getResourceTimestampImpl(String resPath) throws IOException {
- File f = file(resPath);
- if (f.exists() && f.isFile())
- return f.lastModified();
- else
- return 0;
- }
-
- @Override
- protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
- File f = file(resPath);
- f.getParentFile().mkdirs();
- FileOutputStream out = new FileOutputStream(f);
- try {
- IOUtils.copy(content, out);
- } finally {
- IOUtils.closeQuietly(out);
- }
-
- f.setLastModified(ts);
- }
-
- @Override
- protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- File f = file(resPath);
- if ((f.exists() && f.lastModified() != oldTS) || (f.exists() == false && oldTS != 0))
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but found " + f.lastModified());
-
- putResourceImpl(resPath, new ByteArrayInputStream(content), newTS);
-
- // some FS lose precision on given time stamp
- return f.lastModified();
- }
-
- @Override
- protected void deleteResourceImpl(String resPath) throws IOException {
- File f = file(resPath);
- f.delete();
- }
-
- @Override
- protected String getReadableResourcePathImpl(String resPath) {
- return file(resPath).toString();
- }
-
- private File file(String resPath) {
- if (resPath.equals("/"))
- return root;
- else
- return new File(root, resPath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
deleted file mode 100644
index 5b8fe54..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class HBaseConnection {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
-
- private static final Map<String, HConnection> ConnPool = new ConcurrentHashMap<String, HConnection>();
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- for (HConnection conn : ConnPool.values()) {
- try {
- conn.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- });
- }
-
- public static void clearCache() {
- ConnPool.clear();
- }
-
- public static HConnection get(String url) {
-
- HConnection connection = ConnPool.get(url);
- try {
- // I don't use DCL since recreate a connection is not a big issue.
- if (connection == null) {
- // find configuration
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- connection = HConnectionManager.createConnection(conf);
- ConnPool.put(url, connection);
- }
- } catch (Throwable t) {
- throw new StorageException("Error when open connection " + url, t);
- }
-
- return connection;
- }
-
- public static void createHTableIfNeeded(String hbaseUrl, String tableName, String... families) throws IOException {
- createHTableIfNeeded(HBaseConnection.get(hbaseUrl), tableName, families);
- }
-
- public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
- try {
- boolean tableExist = false;
- try {
- hbase.getTableDescriptor(TableName.valueOf(tableName));
- tableExist = true;
- } catch (TableNotFoundException e) {
- }
-
- if (tableExist) {
- logger.debug("HTable '" + tableName + "' already exists");
- return;
- }
-
- logger.debug("Creating HTable '" + tableName + "'");
-
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
-
- if (null != families && families.length > 0) {
- for (String family : families) {
- HColumnDescriptor fd = new HColumnDescriptor(family);
- fd.setInMemory(true); // metadata tables are best in memory
- desc.addFamily(fd);
- }
- }
- hbase.createTable(desc);
-
- logger.debug("HTable '" + tableName + "' created");
- } finally {
- hbase.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
deleted file mode 100644
index d1ff27a..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HadoopUtil;
-
-import com.google.common.collect.Lists;
-
-public class HBaseResourceStore extends ResourceStore {
-
- private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
- private static final String FAMILY = "f";
- private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
- private static final String COLUMN = "c";
- private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
- private static final String COLUMN_TS = "t";
- private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
-
- private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
-
- static {
- TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
- TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
- TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
- TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
- TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
- TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
- TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
- TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
- }
-
- final String tableNameBase;
- final String hbaseUrl;
-
- // final Map<String, String> tableNameMap; // path prefix ==> HBase table name
-
- private HConnection getConnection() throws IOException {
- return HBaseConnection.get(hbaseUrl);
- }
-
- public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
- super(kylinConfig);
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
- hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
- if (!(StringUtils.isEmpty(hbaseUrl) || "hbase".equals(hbaseUrl)))
- throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.metadata.url=kylin_metadata@hbase' in kylin.properties");
-
- createHTableIfNeeded(getAllInOneTableName());
-
- }
-
- private void createHTableIfNeeded(String tableName) throws IOException {
- HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
- }
-
- private String getAllInOneTableName() {
- return tableNameBase;
- }
-
- @Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
- assert resPath.startsWith("/");
- String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
- byte[] startRow = Bytes.toBytes(lookForPrefix);
- byte[] endRow = Bytes.toBytes(lookForPrefix);
- endRow[endRow.length - 1]++;
-
- ArrayList<String> result = new ArrayList<String>();
-
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- Scan scan = new Scan(startRow, endRow);
- scan.setFilter(new KeyOnlyFilter());
- try {
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- String path = Bytes.toString(r.getRow());
- assert path.startsWith(lookForPrefix);
- int cut = path.indexOf('/', lookForPrefix.length());
- String child = cut < 0 ? path : path.substring(0, cut);
- if (result.contains(child) == false)
- result.add(child);
- }
- } finally {
- IOUtils.closeQuietly(table);
- }
- // return null to indicate not a folder
- return result.isEmpty() ? null : result;
- }
-
- @Override
- protected boolean existsImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, false, false);
- return r != null;
- }
-
- @Override
- protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException {
- byte[] startRow = Bytes.toBytes(rangeStart);
- byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
-
- Scan scan = new Scan(startRow, endRow);
- scan.addColumn(B_FAMILY, B_COLUMN_TS);
- scan.addColumn(B_FAMILY, B_COLUMN);
- tuneScanParameters(scan);
-
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- List<RawResource> result = Lists.newArrayList();
- try {
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
- }
- } catch (IOException e) {
- for (RawResource rawResource : result) {
- IOUtils.closeQuietly(rawResource.inputStream);
- }
- throw e;
- } finally {
- IOUtils.closeQuietly(table);
- }
- return result;
- }
-
- private void tuneScanParameters(Scan scan) {
- // divide by 10 as some resource like dictionary or snapshot can be very large
- scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10);
- scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize());
- scan.setCacheBlocks(true);
- }
-
- private InputStream getInputStream(String resPath, Result r) throws IOException {
- if (r == null) {
- return null;
- }
- byte[] value = r.getValue(B_FAMILY, B_COLUMN);
- if (value.length == 0) {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- return fileSystem.open(redirectPath);
- } else {
- return new ByteArrayInputStream(value);
- }
- }
-
- private long getTimestamp(Result r) {
- if (r == null || r.getValue(B_FAMILY, B_COLUMN_TS) == null) {
- return 0;
- } else {
- return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
- }
- }
-
- @Override
- protected RawResource getResourceImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, true, true);
- if (r == null)
- return null;
- else
- return new RawResource(getInputStream(resPath, r), getTimestamp(r));
- }
-
- @Override
- protected long getResourceTimestampImpl(String resPath) throws IOException {
- return getTimestamp(getByScan(resPath, false, true));
- }
-
- @Override
- protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(content, bout);
- bout.close();
-
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- try {
- byte[] row = Bytes.toBytes(resPath);
- Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
-
- table.put(put);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- try {
- byte[] row = Bytes.toBytes(resPath);
- byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
- Put put = buildPut(resPath, newTS, row, content, table);
-
- boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
- if (!ok) {
- long real = getResourceTimestampImpl(resPath);
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + real + ", but it is " + oldTS);
- }
-
- table.flushCommits();
-
- return newTS;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- try {
- Delete del = new Delete(Bytes.toBytes(resPath));
- table.delete(del);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected String getReadableResourcePathImpl(String resPath) {
- return getAllInOneTableName() + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
- }
-
- private Result getByScan(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- byte[] startRow = Bytes.toBytes(path);
- byte[] endRow = plusZero(startRow);
-
- Scan scan = new Scan(startRow, endRow);
- if (!fetchContent && !fetchTimestamp) {
- scan.setFilter(new KeyOnlyFilter());
- } else {
- if (fetchContent)
- scan.addColumn(B_FAMILY, B_COLUMN);
- if (fetchTimestamp)
- scan.addColumn(B_FAMILY, B_COLUMN_TS);
- }
-
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- try {
- ResultScanner scanner = table.getScanner(scan);
- Result result = null;
- for (Result r : scanner) {
- result = r;
- }
- return result == null || result.isEmpty() ? null : result;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private byte[] plusZero(byte[] startRow) {
- byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
- endRow[endRow.length - 1] = 0;
- return endRow;
- }
-
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- if (fileSystem.exists(redirectPath)) {
- fileSystem.delete(redirectPath, true);
- }
-
- FSDataOutputStream out = fileSystem.create(redirectPath);
-
- try {
- out.write(largeColumn);
- } finally {
- IOUtils.closeQuietly(out);
- }
-
- return redirectPath;
- }
-
- public Path bigCellHDFSPath(String resPath) {
- String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
- Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
- return redirectPath;
- }
-
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
- int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
- if (content.length > kvSizeLimit) {
- writeLargeCellToHdfs(resPath, content, table);
- content = BytesUtil.EMPTY_BYTE_ARRAY;
- }
-
- Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
-
- return put;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java b/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
deleted file mode 100644
index a23b933..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.kylin.common.util.JsonUtil;
-
-/**
- * @author yangli9
- */
-public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
-
- Class<T> clz;
-
- public JsonSerializer(Class<T> clz) {
- this.clz = clz;
- }
-
- @Override
- public T deserialize(DataInputStream in) throws IOException {
- return JsonUtil.readValue(in, clz);
- }
-
- @Override
- public void serialize(T obj, DataOutputStream out) throws IOException {
- JsonUtil.writeValueIndent(out, obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java b/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
deleted file mode 100644
index 4f52553..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/RawResource.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.InputStream;
-
-/**
- */
-public class RawResource {
-
- public final InputStream inputStream;
- public final long timestamp;
-
- public RawResource(InputStream inputStream, long timestamp) {
- this.inputStream = inputStream;
- this.timestamp = timestamp;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
deleted file mode 100644
index 505c72a..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-abstract public class ResourceStore {
-
- private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
-
- public static final String CUBE_RESOURCE_ROOT = "/cube";
- public static final String II_RESOURCE_ROOT = "/invertedindex";
- public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
- public static final String II_DESC_RESOURCE_ROOT = "/invertedindex_desc";
- public static final String DATA_MODEL_DESC_RESOURCE_ROOT = "/model_desc";
- public static final String DICT_RESOURCE_ROOT = "/dict";
- public static final String JOB_PATH_ROOT = "/job";
- public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
- public static final String PROJECT_RESOURCE_ROOT = "/project";
- public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
- public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
- public static final String TABLE_RESOURCE_ROOT = "/table";
- public static final String HYBRID_RESOURCE_ROOT = "/hybrid";
- public static final String EXECUTE_PATH_ROOT = "/execute";
- public static final String EXECUTE_OUTPUT_ROOT = "/execute_output";
-
-
- private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
-
- public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
-
- static {
- knownImpl.add(FileResourceStore.class);
- knownImpl.add(HBaseResourceStore.class);
- }
-
- public static ResourceStore getStore(KylinConfig kylinConfig) {
- ResourceStore r = CACHE.get(kylinConfig);
- List<Throwable> es = new ArrayList<Throwable>();
- if (r == null) {
- logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
- for (Class<? extends ResourceStore> cls : knownImpl) {
-
- try {
- r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
- } catch (Exception e) {
- es.add(e);
- } catch (NoClassDefFoundError er) {
- // may throw NoClassDefFoundError
- es.add(er);
- }
- if (r != null) {
- break;
- }
- }
- if (r == null) {
- for (Throwable exceptionOrError : es) {
- logger.error("Create new store instance failed ", exceptionOrError);
- }
- throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
- }
-
- CACHE.put(kylinConfig, r);
- }
- return r;
- }
-
- // ============================================================================
-
- KylinConfig kylinConfig;
-
- ResourceStore(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- }
-
- /**
- * return a list of child resources & folders under given path, return null
- * if given path is not a folder
- */
- final public ArrayList<String> listResources(String resPath) throws IOException {
- resPath = norm(resPath);
- return listResourcesImpl(resPath);
- }
-
- abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
-
- /**
- * return true if a resource exists, return false in case of folder or
- * non-exist
- */
- final public boolean exists(String resPath) throws IOException {
- return existsImpl(norm(resPath));
- }
-
- abstract protected boolean existsImpl(String resPath) throws IOException;
-
- /**
- * read a resource, return null in case of not found
- */
- final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- RawResource res = getResourceImpl(resPath);
- if (res == null)
- return null;
-
- DataInputStream din = new DataInputStream(res.inputStream);
- try {
- T r = serializer.deserialize(din);
- r.setLastModified(res.timestamp);
- return r;
- } finally {
- IOUtils.closeQuietly(din);
- IOUtils.closeQuietly(res.inputStream);
- }
- }
-
- final public RawResource getResource(String resPath) throws IOException {
- return getResourceImpl(norm(resPath));
- }
-
- final public long getResourceTimestamp(String resPath) throws IOException {
- return getResourceTimestampImpl(norm(resPath));
- }
-
- final public <T extends RootPersistentEntity> List<T> getAllResources(String rangeStart, String rangeEnd, Class<T> clazz, Serializer<T> serializer) throws IOException {
- final List<RawResource> allResources = getAllResources(rangeStart, rangeEnd);
- if (allResources.isEmpty()) {
- return Collections.emptyList();
- }
- List<T> result = Lists.newArrayList();
- try {
- for (RawResource rawResource : allResources) {
- final T element = serializer.deserialize(new DataInputStream(rawResource.inputStream));
- element.setLastModified(rawResource.timestamp);
- result.add(element);
- }
- return result;
- } finally {
- for (RawResource rawResource : allResources) {
- IOUtils.closeQuietly(rawResource.inputStream);
- }
- }
- }
-
- abstract protected List<RawResource> getAllResources(String rangeStart, String rangeEnd) throws IOException;
-
- /** returns null if not exists */
- abstract protected RawResource getResourceImpl(String resPath) throws IOException;
-
- /** returns 0 if not exists */
- abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
-
- /**
- * overwrite a resource without write conflict check
- */
- final public void putResource(String resPath, InputStream content, long ts) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- putResourceImpl(resPath, content, ts);
- }
-
- abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
-
- /**
- * check & set, overwrite a resource
- */
- final public <T extends RootPersistentEntity> long putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
-
- long oldTS = obj.getLastModified();
- long newTS = System.currentTimeMillis();
- obj.setLastModified(newTS);
-
- try {
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- DataOutputStream dout = new DataOutputStream(buf);
- serializer.serialize(obj, dout);
- dout.close();
- buf.close();
-
- newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
- obj.setLastModified(newTS); // update again the confirmed TS
- return newTS;
- } catch (IOException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- } catch (RuntimeException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- }
- }
-
- /**
- * checks old timestamp when overwriting existing
- */
- abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
-
- /**
- * delete a resource, does nothing on a folder
- */
- final public void deleteResource(String resPath) throws IOException {
- logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- deleteResourceImpl(norm(resPath));
- }
-
- abstract protected void deleteResourceImpl(String resPath) throws IOException;
-
- /**
- * get a readable string of a resource path
- */
- final public String getReadableResourcePath(String resPath) {
- return getReadableResourcePathImpl(norm(resPath));
- }
-
- abstract protected String getReadableResourcePathImpl(String resPath);
-
- private String norm(String resPath) {
- resPath = resPath.trim();
- while (resPath.startsWith("//"))
- resPath = resPath.substring(1);
- while (resPath.endsWith("/"))
- resPath = resPath.substring(0, resPath.length() - 1);
- if (resPath.startsWith("/") == false)
- resPath = "/" + resPath;
- return resPath;
- }
-
- // ============================================================================
-
- public static interface Visitor {
- void visit(String path) throws IOException;
- }
-
- public void scanRecursively(String path, Visitor visitor) throws IOException {
- ArrayList<String> children = listResources(path);
- if (children != null) {
- for (String child : children)
- scanRecursively(child, visitor);
- return;
- }
-
- if (exists(path))
- visitor.visit(path);
- }
-
- public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
- final ArrayList<String> collector = Lists.newArrayList();
- scanRecursively(root, new Visitor() {
- @Override
- public void visit(String path) {
- if (path.endsWith(suffix))
- collector.add(path);
- }
- });
- return collector;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
deleted file mode 100644
index 0ebed3d..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.StringUtil;
-
-public class ResourceTool {
-
- private static String[] excludes = null;
-
- public static void main(String[] args) throws IOException {
- args = StringUtil.filterSystemArgs(args);
-
- if (args.length == 0) {
- System.out.println("Usage: MetadataTool reset");
- System.out.println("Usage: MetadataTool list RESOURCE_PATH");
- System.out.println("Usage: MetadataTool download LOCAL_DIR");
- System.out.println("Usage: MetadataTool upload LOCAL_DIR");
- return;
- }
-
- String exclude = System.getProperty("exclude");
- if (exclude != null) {
- excludes = exclude.split("\\s*,\\s*");
- }
-
- String cmd = args[0];
- switch (cmd) {
- case "reset":
- reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
- break;
- case "list":
- list(KylinConfig.getInstanceFromEnv(), args[1]);
- break;
- case "download":
- copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
- break;
- case "upload":
- copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
- break;
- case "remove":
- remove(KylinConfig.getInstanceFromEnv(), args[1]);
- break;
- default:
- System.out.println("Unknown cmd: " + cmd);
- }
- }
-
- public static void list(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- ArrayList<String> result = store.listResources(path);
- System.out.println("" + result);
- }
-
- public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
-
- ResourceStore src = ResourceStore.getStore(srcConfig);
- ResourceStore dst = ResourceStore.getStore(dstConfig);
- copyR(src, dst, "/");
- }
-
- private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
- ArrayList<String> children = src.listResources(path);
-
- // case of resource (not a folder)
- if (children == null) {
- if (matchExclude(path) == false) {
- RawResource res = src.getResource(path);
- if (res != null) {
- dst.putResource(path, res.inputStream, res.timestamp);
- res.inputStream.close();
- } else {
- System.out.println("Resource not exist for " + path);
- }
- }
- }
- // case of folder
- else {
- for (String child : children)
- copyR(src, dst, child);
- }
- }
-
- private static boolean matchExclude(String path) {
- if (excludes == null)
- return false;
- for (String exclude : excludes) {
- if (path.startsWith(exclude))
- return true;
- }
- return false;
- }
-
- public static void reset(KylinConfig config) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, "/");
- }
-
- private static void resetR(ResourceStore store, String path) throws IOException {
- ArrayList<String> children = store.listResources(path);
- if (children == null) { // path is a resource (not a folder)
- if (matchExclude(path) == false) {
- store.deleteResource(path);
- }
- } else {
- for (String child : children)
- resetR(store, child);
- }
- }
-
- private static void remove(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, path);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
deleted file mode 100644
index c6f3f55..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.UUID;
-
-import org.apache.commons.lang.time.FastDateFormat;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Marks the root entity of JSON persistence. Unit of read, write, cache, and
- * refresh.
- *
- * - CubeInstance - CubeDesc - SourceTable - JobMeta - Dictionary (not JSON but
- * also top level persistence)
- *
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-abstract public class RootPersistentEntity implements AclEntity {
-
- static final String DATE_PATTERN = "yyyy-MM-dd HH:mm:ss z";
- static FastDateFormat format = FastDateFormat.getInstance(DATE_PATTERN);
- static DateFormat df = new SimpleDateFormat(DATE_PATTERN);
-
- public static String formatTime(long millis) {
- return format.format(millis);
- }
-
- public static long parseTime(String timeString) {
- if (timeString == null)
- return 0;
- try {
- Date dt = df.parse(timeString);
- return dt.getTime();
- } catch (ParseException e) {
- }
- return 0l;
- }
-
- // ============================================================================
-
- @JsonProperty("uuid")
- protected String uuid;
-
- @JsonProperty("last_modified")
- protected long lastModified;
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String uuid) {
- this.uuid = uuid;
- }
-
- public String getId() {
- return uuid;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- public void setLastModified(long lastModified) {
- this.lastModified = lastModified;
- }
-
- public void updateRandomUuid() {
- setUuid(UUID.randomUUID().toString());
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (lastModified ^ (lastModified >>> 32));
- result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RootPersistentEntity other = (RootPersistentEntity) obj;
- if (lastModified != other.lastModified)
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/Serializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/Serializer.java b/common/src/main/java/org/apache/kylin/common/persistence/Serializer.java
deleted file mode 100644
index 6861f6a..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/Serializer.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * @author yangli9
- *
- */
-public interface Serializer<T extends RootPersistentEntity> {
-
- public void serialize(T obj, DataOutputStream out) throws IOException;
-
- public T deserialize(DataInputStream in) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/persistence/StorageException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/StorageException.java b/common/src/main/java/org/apache/kylin/common/persistence/StorageException.java
deleted file mode 100644
index 8e2e183..0000000
--- a/common/src/main/java/org/apache/kylin/common/persistence/StorageException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-/**
- *
- * @author xjiang
- *
- */
-public class StorageException extends RuntimeException {
-
- private static final long serialVersionUID = -3748712888242406257L;
-
- public StorageException(String msg, Throwable t) {
- super(msg, t);
- }
-
- public StorageException(String msg) {
- super(msg);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java b/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
deleted file mode 100644
index 679ef14..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-/**
- * @author xjiang
- *
- */
-public abstract class AbstractRestCache<K, V> {
-
- protected final Broadcaster.TYPE syncType;
-
- protected AbstractRestCache(Broadcaster.TYPE syncType) {
- this.syncType = syncType;
- }
-
- protected final void syncRemote(K key, Broadcaster.EVENT syncAction) {
- Broadcaster.getInstance().queue(syncType.getType(), syncAction.getType(), key.toString());
- }
-
- public abstract void put(K key, V value);
-
- public abstract void putLocal(K key, V value);
-
- public abstract void remove(K key);
-
- public abstract void removeLocal(K key);
-
- public abstract void clear();
-
- public abstract int size();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
deleted file mode 100644
index 7978d20..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
-/**
- * Broadcast kylin event out
- *
- * @author jianliu
- *
- */
-public class Broadcaster {
-
- private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
-
- private BlockingDeque<BroadcastEvent> broadcastEvents = new LinkedBlockingDeque<>();
-
- private AtomicLong counter = new AtomicLong();
-
- static class BroadcasterHolder {
- static final Broadcaster INSTANCE = new Broadcaster();
- }
-
- private Broadcaster() {
- Executors.newSingleThreadExecutor().execute(new Runnable() {
- @Override
- public void run() {
- final String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
- if (nodes == null || nodes.length < 1) {//TODO if the node count is greater than 1, it means it is a cluster
- logger.warn("There is no available rest server; check the 'kylin.rest.servers' config");
- return;
- }
- logger.debug(nodes.length + " nodes in the cluster: " + Arrays.toString(nodes));
- final List<RestClient> restClients = Lists.newArrayList();
- for (String node : nodes) {
- restClients.add(new RestClient(node));
- }
- final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size());
- while (true) {
- try {
- final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
- logger.info("new broadcast event:" + broadcastEvent);
- for (final RestClient restClient : restClients) {
- wipingCachePool.execute(new Runnable() {
- @Override
- public void run() {
- try {
- restClient.wipeCache(broadcastEvent.getType(), broadcastEvent.getAction(), broadcastEvent.getName());
- } catch (IOException e) {
- logger.warn("Thread failed during wipe cache at " + broadcastEvent);
- }
- }
- });
- }
- } catch (Exception e) {
- logger.error("error running wiping", e);
- }
- }
- }
- });
- }
-
- public static Broadcaster getInstance() {
- return BroadcasterHolder.INSTANCE;
- }
-
- /**
- * Broadcast the cubedesc event out
- *
- * @param action
- * event action
- */
- public void queue(String type, String action, String key) {
- try {
- counter.incrementAndGet();
- broadcastEvents.putFirst(new BroadcastEvent(type, action, key));
- } catch (Exception e) {
- counter.decrementAndGet();
- logger.error("error putting BroadcastEvent", e);
- }
- }
-
- public long getCounterAndClear() {
- return counter.getAndSet(0);
- }
-
- public static enum EVENT {
- CREATE("create"), UPDATE("update"), DROP("drop");
- private String text;
-
- private EVENT(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- public static EVENT getEvent(String event) {
- for (EVENT one : values()) {
- if (one.getType().equalsIgnoreCase(event)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public static enum TYPE {
- ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model"), HYBRID("hybrid");
- private String text;
-
- private TYPE(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- /**
- * @param type
- * @return
- */
- public static TYPE getType(String type) {
- for (TYPE one : values()) {
- if (one.getType().equalsIgnoreCase(type)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public static class BroadcastEvent {
- private String type;
- private String action;
- private String name;
-
- public BroadcastEvent(String type, String action, String name) {
- super();
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- public String getType() {
- return type;
- }
-
- public String getAction() {
- return action;
- }
-
- public String getName() {
- return name;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((action == null) ? 0 : action.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((type == null) ? 0 : type.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (this == obj) {
- return true;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- BroadcastEvent other = (BroadcastEvent) obj;
- if (!StringUtils.equals(action, other.action)) {
- return false;
- }
- if (!StringUtils.equals(name, other.name)) {
- return false;
- }
- if (!StringUtils.equals(type, other.type)) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("type", type).add("name", name).add("action", action).toString();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java b/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
deleted file mode 100644
index c29e7b1..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/CaseInsensitiveStringCache.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.util.concurrent.ConcurrentSkipListMap;
-
-/**
- * Created by qianzhou on 1/15/15.
- */
-public class CaseInsensitiveStringCache<V> extends SingleValueCache<String, V> {
-
- public CaseInsensitiveStringCache(Broadcaster.TYPE syncType) {
- super(syncType, new ConcurrentSkipListMap<String, V>(String.CASE_INSENSITIVE_ORDER));
- }
-
- @Override
- public void put(String key, V value) {
- super.put(key, value);
- }
-
- @Override
- public void putLocal(String key, V value) {
- super.putLocal(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
deleted file mode 100644
index ca7acd2..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
deleted file mode 100644
index 58b1713..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.httpclient.Credentials;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.kylin.common.util.Bytes;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-/**
- * @author yangli9
- */
-public class RestClient {
-
- String host;
- int port;
- String baseUrl;
- String userName;
- String password;
- HttpClient client;
-
- private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
-
- public static boolean matchFullRestPattern(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- return m.matches();
- }
-
- /**
- * @param uri
- * "user:pwd@host:port"
- */
- public RestClient(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- if (!m.matches())
- throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern);
-
- String user = m.group(1);
- String pwd = m.group(2);
- String host = m.group(3);
- String portStr = m.group(4);
- int port = Integer.parseInt(portStr == null ? "7070" : portStr);
-
- init(host, port, user, pwd);
- }
-
- private void init(String host, int port, String userName, String password) {
- this.host = host;
- this.port = port;
- this.userName = userName;
- this.password = password;
- this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
-
- client = new HttpClient();
-
- if (userName != null && password != null) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials creds = new UsernamePasswordCredentials(userName, password);
- client.getState().setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM), creds);
- }
- }
-
- public void wipeCache(String type, String action, String name) throws IOException {
- String url = baseUrl + "/cache/" + type + "/" + name + "/" + action;
- HttpMethod request = new PutMethod(url);
-
- try {
- int code = client.executeMethod(request);
- String msg = Bytes.toString(request.getResponseBody());
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- } catch (HttpException ex) {
- throw new IOException(ex);
- } finally {
- request.releaseConnection();
- }
- }
-
- public String getKylinProperties() throws IOException {
- String url = baseUrl + "/admin/config";
- HttpMethod request = new GetMethod(url);
- try {
- int code = client.executeMethod(request);
- String msg = Bytes.toString(request.getResponseBody());
- JSONObject obj = new JSONObject(msg);
- msg = obj.getString("config");
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- return msg;
-
- } catch (JSONException e) {
- throw new IOException("Error when parsing json response from REST");
- } finally {
- request.releaseConnection();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
deleted file mode 100644
index 55af75e..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-/**
- * @author xjiang
- *
- */
-public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final ConcurrentMap<K, V> innerCache;
-
- public SingleValueCache(Broadcaster.TYPE syncType) {
- this(syncType, new ConcurrentHashMap<K, V>());
- }
-
- public SingleValueCache(Broadcaster.TYPE syncType, ConcurrentMap<K, V> innerCache) {
- super(syncType);
- this.innerCache = innerCache;
- }
-
- public void put(K key, V value) {
- final V result = innerCache.put(key, value);
- if (result == null) {
- syncRemote(key, Broadcaster.EVENT.CREATE);
- } else {
- syncRemote(key, Broadcaster.EVENT.UPDATE);
- }
- }
-
- public void putLocal(K key, V value) {
- innerCache.put(key, value);
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.remove(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- innerCache.remove(key);
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public V get(K key) {
- return innerCache.get(key);
- }
-
- public Collection<V> values() {
- return innerCache.values();
- }
-
- public boolean containsKey(String key) {
- return innerCache.containsKey(key);
- }
-
- public Map<K, V> getMap() {
- return Collections.unmodifiableMap(innerCache);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Array.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Array.java b/common/src/main/java/org/apache/kylin/common/util/Array.java
deleted file mode 100644
index 26cd597..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Array.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Arrays;
-
-/*
- * An array with correct equals(), hashCode(), compareTo() and toString()
- */
-public class Array<T> implements Comparable<Array<T>> {
- public T[] data;
-
- public Array(T[] data) {
- this.data = data;
- }
-
- public String toString() {
- return Arrays.toString(data);
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof Array) {
- return Arrays.equals(this.data, ((Array<?>) o).data);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(data);
- }
-
- @Override
- public int compareTo(Array<T> other) {
- return compare(this.data, other.data, null);
- }
-
- @SuppressWarnings("unchecked")
- public static <T> int compare(T[] a, T[] b, boolean[] ascending) {
- int r = 0;
- int n = Math.min(a.length, b.length);
- boolean asc = true;
-
- for (int i = 0; i < n; i++) {
- r = ((Comparable<T>) a[i]).compareTo(b[i]);
- if (r != 0) {
- asc = (ascending != null && ascending.length > i) ? ascending[i] : true;
- return asc ? r : -r;
- }
- }
- return a.length - b.length;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
deleted file mode 100644
index 92e0da2..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Arrays;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable<ByteArray> {
-
- public byte[] data;
-
- public ByteArray(byte[] data) {
- this.data = data;
- }
-
- @Override
- public int hashCode() {
- return Bytes.hashCode(data);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ByteArray other = (ByteArray) obj;
- if (!Arrays.equals(data, other.data))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(ByteArray o) {
- return Bytes.compareTo(this.data, o.data);
- }
-}
[03/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
deleted file mode 100644
index 3f4c857..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configuration.IntegerRanges;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.MapContext;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
-import org.apache.hadoop.security.Credentials;
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext {
-
- public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
-
- hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
-
- return new WrappedMapper().getMapContext(new MapContext() {
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentKey() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public void write(Object key, Object value) throws IOException, InterruptedException {
- System.out.println("Write -- k:" + key + ", v:" + value);
- if (outKV != null) {
- outKV[0] = key;
- outKV[1] = value;
- }
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- throw new NotImplementedException();
- }
-
- @Override
- public TaskAttemptID getTaskAttemptID() {
- throw new NotImplementedException();
- }
-
- @Override
- public void setStatus(String msg) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getStatus() {
- throw new NotImplementedException();
- }
-
- @Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(Enum<?> counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(String groupName, String counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hconf;
- }
-
- @Override
- public Credentials getCredentials() {
- throw new NotImplementedException();
- }
-
- @Override
- public JobID getJobID() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getNumReduceTasks() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path getWorkingDirectory() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJobName() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getSortComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJar() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getGroupingComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getJobSetupCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getProfileEnabled() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getProfileParams() {
- throw new NotImplementedException();
- }
-
- @Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getUser() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getSymlink() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getArchiveClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getFileClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getArchiveTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getFileTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxMapAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxReduceAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public void progress() {
- throw new NotImplementedException();
- }
-
- @Override
- public InputSplit getInputSplit() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getCombinerKeyGroupingComparator() {
- throw new NotImplementedException();
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
deleted file mode 100644
index 8b5e78d..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class NDCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob8D() throws Exception {
- String input = "src/test/resources/data/base_cuboid/";
- String output = "target/test-output/8d_cuboid";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "8d_cuboid";
- String level = "1";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-
- @Test
- public void testJob7D() throws Exception {
- final String input = "src/test/resources/data/8d_cuboid/";
- final String output = "target/test-output/7d_cuboid";
- final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "7d_cuboid";
- String level = "2";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
deleted file mode 100644
index d60ec67..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.annotation.Nullable;
-
-public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
- MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- NDCuboidMapper mapper = new NDCuboidMapper();
- CuboidReducer reducer = new CuboidReducer();
- mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapReduceWithSlr() throws IOException {
-
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
- Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));
-
- mapReduceDriver.addInput(input1);
-
- List<Pair<Text, Text>> result = mapReduceDriver.run();
-
- assertEquals(4, result.size());
-
- byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
- Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));
-
-
- //As we will truncate decimal(KYLIN-766), value will no longer equals to resultValue
- Collection<Text> keys = Collections2.transform(result, new Function<Pair<Text, Text>, Text>() {
- @Nullable
- @Override
- public Text apply(Pair<Text, Text> input) {
- return input.getFirst();
- }
- });
- assertTrue(keys.contains(output1.getFirst()));
- assertTrue(!result.contains(output1));
-
- long[] keySet = new long[result.size()];
-
- System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 }));
- for (int i = 0; i < result.size(); i++) {
- byte[] bytes = new byte[result.get(i).getFirst().getLength()];
- System.arraycopy(result.get(i).getFirst().getBytes(), 0, bytes, 0, result.get(i).getFirst().getLength());
- System.out.println(Bytes.toLong(bytes));
- keySet[i] = Bytes.toLong(bytes);
- }
-
- // refer to CuboidSchedulerTest.testGetSpanningCuboid()
- assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
deleted file mode 100644
index 9a6fbbb..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.invertedindex.RandomKeyDistributionMapper;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionMapperTest {
-
- MapDriver<Text, Text, Text, LongWritable> mapDriver;
-
- @Before
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void setUp() {
- RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @Test
- public void test() throws IOException {
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1001; i++) {
- data.add(new Text(String.valueOf(i)));
- }
-
- for (Text t : data) {
- mapDriver.addInput(t, new Text("abc"));
- }
-
- mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100");
- List<Pair<Text, LongWritable>> result = mapDriver.run();
- assertEquals(100, result.size());
-
- for (Pair<Text, LongWritable> p : result) {
- System.out.println(p.getFirst());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java
deleted file mode 100644
index e5f0b7e..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.invertedindex.RandomKeyDistributionReducer;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionReducerTest {
- ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
-
- @Before
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void setUp() {
- RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @Test
- public void test() throws IOException {
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1001; i++) {
- data.add(new Text(String.valueOf(i)));
- }
- for (Text t : data) {
- reduceDriver.addInput(t, new ArrayList<LongWritable>());
- }
-
- reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2");
- List<Pair<Text, LongWritable>> result = reduceDriver.run();
-
- assertEquals(2, result.size());
-
- for (Pair<Text, LongWritable> p : result) {
- System.out.println(p.getFirst());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
deleted file mode 100644
index 8e0c3bb..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/8d_cuboid/";
- String output = "target/test-output/key_distribution_range/";
- String jobname = "calculate_splits";
- String cubename = "test_kylin_cube_with_slr_ready";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-output", output, "-jobname", jobname, "-cubename", cubename };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new RangeKeyDistributionJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java
deleted file mode 100644
index 6cc8703..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionMapperTest {
-
- @SuppressWarnings("rawtypes")
- MapDriver mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() {
- RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapperWithoutHeader() throws IOException {
-
- Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-
- mapDriver.addInput(inputKey1, new Text("abc"));
- mapDriver.addInput(inputKey2, new Text("abc"));
- mapDriver.addInput(inputKey3, new Text("abc"));
- mapDriver.addInput(inputKey4, new Text("abc"));
- mapDriver.addInput(inputKey5, new Text("abc"));
- mapDriver.addInput(inputKey6, new Text("abc"));
- mapDriver.addInput(inputKey7, new Text("abc"));
-
- List<Pair<Text, LongWritable>> result = mapDriver.run();
-
- assertEquals(1, result.size());
-
- byte[] key1 = result.get(0).getFirst().getBytes();
- LongWritable value1 = result.get(0).getSecond();
- assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
- assertEquals(147, value1.get());
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapperWithHeader() throws IOException {
-
- Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-
- mapDriver.addInput(inputKey1, new Text("abc"));
- mapDriver.addInput(inputKey2, new Text("abc"));
- mapDriver.addInput(inputKey3, new Text("abc"));
- mapDriver.addInput(inputKey4, new Text("abc"));
- mapDriver.addInput(inputKey5, new Text("abc"));
- mapDriver.addInput(inputKey6, new Text("abc"));
- mapDriver.addInput(inputKey7, new Text("abc"));
-
- List<Pair<Text, LongWritable>> result = mapDriver.run();
-
- assertEquals(1, result.size());
-
- byte[] key1 = result.get(0).getFirst().getBytes();
- LongWritable value1 = result.get(0).getSecond();
- assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
- assertEquals(273, value1.get());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java
deleted file mode 100644
index c1480a0..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionReducerTest {
-
- ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() {
- RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @Test
- public void testReducer() throws IOException {
- // TODO
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java
deleted file mode 100644
index 365a0d8..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hbase;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.hadoop.cube.BaseCuboidJobTest;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CreateHTableTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
-
- this.createTestMetadata();
-
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testGetSplits() throws IllegalArgumentException, Exception {
- CreateHTableJob c = new CreateHTableJob();
-
- String input = "src/test/resources/partition_list/part-r-00000";
-
- byte[][] splits = c.getSplits(conf, new Path(input));
-
- assertEquals(497, splits.length);
- assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 15, -1, 11, 51, -45, 2 }, splits[0]);
- assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 3, -1, -1, -54, -61, 109, -44, 1 }, splits[496]);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
deleted file mode 100644
index f2b9ed6..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * Created by hongbin on 5/15/14.
- */
-public class TestHbaseClient {
-
- private static boolean reverse = false;
-
- public static void foo(int n, int k) {
- int t = k;
- if (n - k < k) {
- t = n - k;
- reverse = true;
- }
- boolean[] flags = new boolean[n];
- inner(flags, 0, t);
- }
-
- private static void print(boolean[] flags) {
- for (int i = 0; i < flags.length; i++) {
- if (!reverse) {
- if (flags[i])
- System.out.print("0");
- else
- System.out.print("1");
- } else {
- if (flags[i])
- System.out.print("1");
- else
- System.out.print("0");
-
- }
- }
- System.out.println();
-
- }
-
- private static void inner(boolean[] flags, int start, int remaining) {
- if (remaining <= 0) {
- print(flags);
- return;
- }
-
- if (flags.length - start < remaining) {
- return;
- }
-
- // write at flags[start]
- flags[start] = true;
- inner(flags, start + 1, remaining - 1);
-
- // not write at flags[start]
- flags[start] = false;
- inner(flags, start + 1, remaining);
- }
-
- public static void main(String[] args) throws IOException {
- foo(6, 5);
- foo(5, 2);
- foo(3, 0);
-
- Configuration conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", "hbase_host");
- conf.set("zookeeper.znode.parent", "/hbase-unsecure");
-
- HTable table = new HTable(conf, "test1");
- Put put = new Put(Bytes.toBytes("row1"));
-
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2"));
-
- table.put(put);
- table.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java
deleted file mode 100644
index 6730f10..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.JoinedFlatTable;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-@Ignore("This test case doesn't have much value, ignore it.")
-public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
-
- CubeInstance cube = null;
- CubeJoinedFlatTableDesc intermediateTableDesc = null;
- String fakeJobUUID = "abc-def";
- CubeSegment cubeSegment = null;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready");
- cubeSegment = cube.getSegments().get(0);
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testGenCreateTableDDL() {
- String ddl = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, "/tmp", fakeJobUUID);
- System.out.println(ddl);
-
- System.out.println("The length for the ddl is " + ddl.length());
- }
-
- @Test
- public void testGenDropTableDDL() {
- String ddl = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, fakeJobUUID);
- System.out.println(ddl);
- assertEquals(107, ddl.length());
- }
-
- @Test
- public void testGenerateInsertSql() throws IOException {
- String sqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
- System.out.println(sqls);
-
- int length = sqls.length();
- assertEquals(1155, length);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
deleted file mode 100644
index f3f4272..0000000
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.impl.threadpool;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.lock.ZookeeperJobLock;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-
-/**
- * Created by qianzhou on 12/26/14.
- */
-public abstract class BaseSchedulerTest extends HBaseMetadataTestCase {
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- static void setFinalStatic(Field field, Object newValue) throws Exception {
- field.setAccessible(true);
-
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- field.set(null, newValue);
- }
-
- protected void waitForJobFinish(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- final ExecutableState status = job.getStatus();
- if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- protected void waitForJobStatus(String jobId, ExecutableState state, long interval) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == state) {
- break;
- } else {
- try {
- Thread.sleep(interval);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"), 10);
- jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new ZookeeperJobLock());
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
deleted file mode 100644
index 9daf87f..0000000
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.impl.threadpool;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.ErrorTestExecutable;
-import org.apache.kylin.job.FailedTestExecutable;
-import org.apache.kylin.job.SelfStopExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.junit.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Created by qianzhou on 12/19/14.
- */
-public class DefaultSchedulerTest extends BaseSchedulerTest {
-
- @Test
- public void testSingleTaskJob() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- }
-
- @Test
- public void testSucceed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState());
- }
-
- @Test
- public void testSucceedAndFailed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new FailedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState());
- }
-
- @Test
- public void testSucceedAndError() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new ErrorTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState());
- }
-
- @Test
- public void testDiscard() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SelfStopExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
- jobService.discardJob(job.getId());
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
- Thread.sleep(5000);
- System.out.println(job);
- }
-
- @Test
- public void testSchedulerPool() throws InterruptedException {
- ScheduledExecutorService fetchPool = Executors.newScheduledThreadPool(1);
- final CountDownLatch countDownLatch = new CountDownLatch(3);
- ScheduledFuture future = fetchPool.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- countDownLatch.countDown();
- }
- }, 5, 5, TimeUnit.SECONDS);
- assertTrue("countDownLatch should reach zero in 15 secs", countDownLatch.await(20, TimeUnit.SECONDS));
- assertTrue("future should still running", future.cancel(true));
-
- final CountDownLatch countDownLatch2 = new CountDownLatch(3);
- ScheduledFuture future2 = fetchPool.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- countDownLatch2.countDown();
- throw new RuntimeException();
- }
- }, 5, 5, TimeUnit.SECONDS);
- assertFalse("countDownLatch2 should NOT reach zero in 15 secs", countDownLatch2.await(20, TimeUnit.SECONDS));
- assertFalse("future2 should has been stopped", future2.cancel(true));
-
- final CountDownLatch countDownLatch3 = new CountDownLatch(3);
- ScheduledFuture future3 = fetchPool.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- countDownLatch3.countDown();
- throw new RuntimeException();
- } catch (Exception e) {
- }
- }
- }, 5, 5, TimeUnit.SECONDS);
- assertTrue("countDownLatch3 should reach zero in 15 secs", countDownLatch3.await(20, TimeUnit.SECONDS));
- assertTrue("future3 should still running", future3.cancel(true));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java b/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
deleted file mode 100644
index d6d5210..0000000
--- a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.manager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.exception.IllegalStateTranferException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ChainedExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.Executable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by qianzhou on 12/16/14.
- */
-public class ExecutableManagerTest extends LocalFileMetadataTestCase {
-
- private ExecutableManager service;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- service = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- for (String jobId : service.getAllJobIds()) {
- System.out.println("deleting " + jobId);
- service.deleteJob(jobId);
- }
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- assertNotNull(service);
- BaseTestExecutable executable = new SucceedTestExecutable();
- executable.setParam("test1", "test1");
- executable.setParam("test2", "test2");
- executable.setParam("test3", "test3");
- service.addJob(executable);
- List<AbstractExecutable> result = service.getAllExecutables();
- assertEquals(1, result.size());
- AbstractExecutable another = service.getJob(executable.getId());
- assertJobEqual(executable, another);
-
- service.updateJobOutput(executable.getId(), ExecutableState.RUNNING, null, "test output");
- assertJobEqual(executable, service.getJob(executable.getId()));
- }
-
- @Test
- public void testDefaultChainedExecutable() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- job.addTask(new SucceedTestExecutable());
- job.addTask(new SucceedTestExecutable());
-
- service.addJob(job);
- assertEquals(2, job.getTasks().size());
- AbstractExecutable anotherJob = service.getJob(job.getId());
- assertEquals(DefaultChainedExecutable.class, anotherJob.getClass());
- assertEquals(2, ((DefaultChainedExecutable) anotherJob).getTasks().size());
- assertJobEqual(job, anotherJob);
- }
-
- @Test
- public void testValidStateTransfer() throws Exception {
- SucceedTestExecutable job = new SucceedTestExecutable();
- String id = job.getId();
- service.addJob(job);
- service.updateJobOutput(id, ExecutableState.RUNNING, null, null);
- service.updateJobOutput(id, ExecutableState.ERROR, null, null);
- service.updateJobOutput(id, ExecutableState.READY, null, null);
- service.updateJobOutput(id, ExecutableState.RUNNING, null, null);
- service.updateJobOutput(id, ExecutableState.READY, null, null);
- service.updateJobOutput(id, ExecutableState.RUNNING, null, null);
- service.updateJobOutput(id, ExecutableState.SUCCEED, null, null);
- }
-
- @Test(expected = IllegalStateTranferException.class)
- public void testInvalidStateTransfer() {
- SucceedTestExecutable job = new SucceedTestExecutable();
- service.addJob(job);
- service.updateJobOutput(job.getId(), ExecutableState.RUNNING, null, null);
- service.updateJobOutput(job.getId(), ExecutableState.STOPPED, null, null);
- }
-
- private static void assertJobEqual(Executable one, Executable another) {
- assertEquals(one.getClass(), another.getClass());
- assertEquals(one.getId(), another.getId());
- assertEquals(one.getStatus(), another.getStatus());
- assertEquals(one.isRunnable(), another.isRunnable());
- assertEquals(one.getOutput(), another.getOutput());
- assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null));
- if (one.getParams() != null) {
- assertEquals(one.getParams().size(), another.getParams().size());
- for (String key : one.getParams().keySet()) {
- assertEquals(one.getParams().get(key), another.getParams().get(key));
- }
- }
- if (one instanceof ChainedExecutable) {
- assertTrue(another instanceof ChainedExecutable);
- List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks();
- List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks();
- assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null));
- if (onesSubs != null) {
- assertEquals(onesSubs.size(), anotherSubs.size());
- for (int i = 0; i < onesSubs.size(); ++i) {
- assertJobEqual(onesSubs.get(i), anotherSubs.get(i));
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java
deleted file mode 100644
index 2c2f16f..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.job.hadoop.cardinality.HiveColumnCardinalityJob;
-import org.apache.kylin.job.hadoop.cube.BaseCuboidJobTest;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-@Ignore("This test is invalid now as the mapper uses HCatalog to fetch the data which need a hive env")
-public class ColumnCardinalityJobTest {
-
- private Configuration conf;
-
- @Before
- public void setup() throws IOException {
- conf = new Configuration();
- BaseCuboidJobTest.setLocalMR(conf);
- }
-
- @Test
- @Ignore("not maintaining")
- public void testJob() throws Exception {
- final String input = "src/test/resources/data/test_cal_dt/";
- final String output = "target/test-output/column-cardinality/";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-output", output, "-cols", "1,2,3,4,5,6,9,0" };
- assertEquals("Job failed", 0, ToolRunner.run(new HiveColumnCardinalityJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
deleted file mode 100644
index 0d96156..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityMapper;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-@Ignore("This test is invalid now as the mapper uses HCatalog to fetch the data which need a hive env")
-public class ColumnCardinalityMapperTest {
-
- @SuppressWarnings("rawtypes")
- MapDriver mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Before
- public void setUp() {
- ColumnCardinalityMapper mapper = new ColumnCardinalityMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- public final static String strArr = "abc,tests,test,test,as,sts,test,tss,sets";
-
- @SuppressWarnings({ "unchecked" })
- @Test
- @Ignore
- public void testMapperOn177() throws IOException {
- mapDriver.clearInput();
- File file = new File("src/test/resources/data/test_cal_dt/part-r-00000");
- FileReader reader = new FileReader(file);
- BufferedReader breader = new BufferedReader(reader);
- String s = breader.readLine();
- int i = 0;
- while (s != null) {
- LongWritable inputKey = new LongWritable(i++);
- mapDriver.addInput(inputKey, new Text(s));
- s = breader.readLine();
- }
- // breader.close();
- List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();
- breader.close();
- assertEquals(9, result.size());
-
- int key1 = result.get(0).getFirst().get();
- BytesWritable value1 = result.get(0).getSecond();
- byte[] bytes = value1.getBytes();
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readRegisters(ByteBuffer.wrap(bytes));
- assertTrue(key1 > 0);
- assertEquals(8, hllc.getCountEstimate());
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapperOnComma() throws IOException {
- mapDriver.clearInput();
- LongWritable inputKey1 = new LongWritable(1);
- LongWritable inputKey2 = new LongWritable(2);
- LongWritable inputKey3 = new LongWritable(3);
- LongWritable inputKey4 = new LongWritable(4);
- LongWritable inputKey5 = new LongWritable(5);
- LongWritable inputKey6 = new LongWritable(6);
- LongWritable inputKey7 = new LongWritable(7);
-
- mapDriver.addInput(inputKey1, new Text());
- mapDriver.addInput(inputKey2, new Text(strArr));
- mapDriver.addInput(inputKey3, new Text(strArr));
- mapDriver.addInput(inputKey4, new Text(strArr));
- mapDriver.addInput(inputKey5, new Text(strArr));
- mapDriver.addInput(inputKey6, new Text(strArr));
- mapDriver.addInput(inputKey7, new Text(strArr));
-
- List<Pair<IntWritable, BytesWritable>> result = mapDriver.run();
-
- assertEquals(9, result.size());
-
- int key1 = result.get(0).getFirst().get();
- BytesWritable value1 = result.get(0).getSecond();
- byte[] bytes = value1.getBytes();
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readRegisters(ByteBuffer.wrap(bytes));
- System.out.println("ab\177ab".length());
- assertTrue(key1 > 0);
- assertEquals(1, hllc.getCountEstimate());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
deleted file mode 100644
index 15d1ffb..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityMapper;
-import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityReducer;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class ColumnCardinalityReducerTest {
-
- ReduceDriver<IntWritable, BytesWritable, IntWritable, LongWritable> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() {
- ColumnCardinalityReducer reducer = new ColumnCardinalityReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- private byte[] getBytes(String str) throws IOException {
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- StringTokenizer tokenizer = new StringTokenizer(str, ColumnCardinalityMapper.DEFAULT_DELIM);
- int i = 0;
- while (tokenizer.hasMoreTokens()) {
- String temp = i + "_" + tokenizer.nextToken();
- i++;
- hllc.add(Bytes.toBytes(temp));
- }
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- return buf.array();
- }
-
- @Test
- public void testReducer() throws IOException {
- IntWritable key1 = new IntWritable(1);
- List<BytesWritable> values1 = new ArrayList<BytesWritable>();
- values1.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr)));
-
- IntWritable key2 = new IntWritable(2);
- List<BytesWritable> values2 = new ArrayList<BytesWritable>();
- values2.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " x")));
-
- IntWritable key3 = new IntWritable(3);
- List<BytesWritable> values3 = new ArrayList<BytesWritable>();
- values3.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xx")));
-
- IntWritable key4 = new IntWritable(4);
- List<BytesWritable> values4 = new ArrayList<BytesWritable>();
- values4.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxx")));
-
- IntWritable key5 = new IntWritable(5);
- List<BytesWritable> values5 = new ArrayList<BytesWritable>();
- values5.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxxx")));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
- reduceDriver.withInput(key4, values4);
- reduceDriver.withInput(key5, values5);
-
- List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();
-
- assertEquals(5, result.size());
-
- int outputKey1 = result.get(0).getFirst().get();
- LongWritable value1 = result.get(0).getSecond();
- assertTrue(outputKey1 == 1);
- assertTrue((10 == value1.get()) || (9 == value1.get()));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java b/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java
deleted file mode 100644
index 61f44d1..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.codehaus.jettison.json.JSONException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by honma on 9/17/14.
- */
-@Ignore("convenient trial tool for dev")
-public class CubeMigrationTests extends LocalFileMetadataTestCase {
- @Before
- public void setup() throws Exception {
- super.createTestMetadata();
- ClassUtil.addClasspath(new File(AbstractKylinTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- }
-
- @After
- public void clean() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testMigrate() throws IOException, JSONException, InterruptedException {
-
- // CubeMigrationCLI.moveCube(KylinConfig.getInstanceFromEnv(),
- // KylinConfig.getInstanceFromEnv(),
- // "test_kylin_cube_with_slr_empty", "migration", "true", "false");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
deleted file mode 100644
index 9f9c23c..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
-@Ignore
-public class HBaseRowDigestTest extends HBaseMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- private static final byte[] CF = "f".getBytes();
- private static final byte[] QN = "c".getBytes();
- static ImmutableBytesWritable k = new ImmutableBytesWritable();
- static ImmutableBytesWritable v = new ImmutableBytesWritable();
-
- @Test
- public static void test() throws IOException {
- String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = null;
- HTableInterface table = null;
- try {
- conn = HBaseConnection.get(hbaseUrl);
- table = conn.getTable("KYLIN_II_YTYWP3CQGJ");
- ResultScanner scanner = table.getScanner(CF, QN);
- StringBuffer sb = new StringBuffer();
- while (true) {
- Result r = scanner.next();
- if (r == null)
- break;
-
- Cell[] cells = r.rawCells();
- Cell c = cells[0];
-
- k.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- v.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
-
- byte[] row = k.copyBytes();
- byte[] value = v.copyBytes();
- // byte[] row = r.getRow();
- // byte[] value = r.getValue(CF, QN);
- //
- sb.append("row length: " + row.length + "\r\n");
- sb.append(BytesUtil.toReadableText(row) + "\r\n");
- sb.append("value length: " + value.length + "\r\n");
- sb.append(BytesUtil.toReadableText(value) + "\r\n");
- }
- System.out.println(sb.toString());
- FileUtils.writeStringToFile(new File("/Users/honma/Desktop/a3"), sb.toString());
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (table != null)
- table.close();
- if (conn != null)
- conn.close();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/8d_cuboid/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/8d_cuboid/part-r-00000 b/job/src/test/resources/data/8d_cuboid/part-r-00000
deleted file mode 100644
index d277125..0000000
Binary files a/job/src/test/resources/data/8d_cuboid/part-r-00000 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/base_cuboid/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/base_cuboid/part-r-00000 b/job/src/test/resources/data/base_cuboid/part-r-00000
deleted file mode 100644
index ed53ffb..0000000
Binary files a/job/src/test/resources/data/base_cuboid/part-r-00000 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/flat_table/000000_0
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/flat_table/000000_0 b/job/src/test/resources/data/flat_table/000000_0
deleted file mode 100644
index 8b1b7cc..0000000
Binary files a/job/src/test/resources/data/flat_table/000000_0 and /dev/null differ
[40/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
deleted file mode 100644
index 7420912..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.v1.CubeDesc;
-import org.apache.kylin.cube.model.v1.CubePartitionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class CubeDescUpgrader {
-
- private String resourcePath;
-
- @SuppressWarnings("unused")
- private static final Log logger = LogFactory.getLog(CubeDescUpgrader.class);
-
- private static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER_V1 = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- public CubeDescUpgrader(String resourcePath) {
- this.resourcePath = resourcePath;
- }
-
- public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
- CubeDesc oldModel = loadOldCubeDesc(resourcePath);
-
- org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
-
- copyUnChangedProperties(oldModel, newModel);
-
- DataModelDesc model = extractDataModel(oldModel, newModel);
- newModel.setModel(model);
-
- updateDimensions(oldModel, newModel);
-
- updateRowkeyDictionary(newModel);
-
- return newModel;
- }
-
- private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) {
-
- DataModelDesc modelDesc = newModel.getModel();
- Map<String, String> pkToFK = Maps.newHashMap();
- for (LookupDesc lookupDesc : modelDesc.getLookups()) {
- if (lookupDesc.getJoin() != null) {
- JoinDesc join = lookupDesc.getJoin();
- for (int i = 0; i < join.getForeignKey().length; i++) {
- pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]);
- }
- }
- }
- RowKeyDesc rowKey = newModel.getRowkey();
-
- for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) {
- if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0)
- rowkeyCol.setDictionary("true");
-
- if (pkToFK.containsKey(rowkeyCol.getColumn())) {
- rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn()));
- }
- }
-
- for (String[] aggregationGroup : rowKey.getAggregationGroups()) {
- for (int i = 0; i < aggregationGroup.length; i++) {
- if (pkToFK.containsKey(aggregationGroup[i])) {
- aggregationGroup[i] = pkToFK.get(aggregationGroup[i]);
- }
- }
- }
-
- }
-
- private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- newModel.setUuid(oldModel.getUuid());
- newModel.setName(oldModel.getName());
- newModel.setDescription(oldModel.getDescription());
- newModel.setNullStrings(oldModel.getNullStrings());
-
- newModel.setMeasures(oldModel.getMeasures());
- newModel.setRowkey(oldModel.getRowkey());
- newModel.setHbaseMapping(oldModel.getHBaseMapping());
-
- newModel.setSignature(oldModel.getSignature());
-
- newModel.setNotifyList(oldModel.getNotifyList());
- newModel.setLastModified(oldModel.getLastModified());
- }
-
- private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) {
- DimensionDesc newDim = new DimensionDesc();
-
- newDim.setId(dimId);
- newDim.setName(name);
- newDim.setTable(getMetadataManager().appendDBName(dim.getTable()));
-
- return newDim;
- }
-
- private void updateDimensions(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<DimensionDesc> newDimensions = Lists.newArrayList();
- newModel.setDimensions(newDimensions);
-
- int dimId = 0;
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
-
- DimensionDesc newDim = null;
- // if a dimension defines "column", "derived" and "hierarchy" at the same time, separate it into three dimensions;
-
- boolean needNameSuffix = false;
- if (dim.getColumn() != null && !"{FK}".equals(dim.getColumn())) {
- //column on fact table
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setColumn(new String[] { dim.getColumn() });
- needNameSuffix = true;
- } else if (ArrayUtils.isEmpty(dim.getDerived()) && ArrayUtils.isEmpty(dim.getHierarchy())) {
- // user defines a lookup table, but didn't use any column other than the pk, in this case, convert to use fact table's fk
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setTable(getMetadataManager().appendDBName(newModel.getFactTable()));
-
- newDim.setColumn(dim.getJoin().getForeignKey());
- }
-
- if (!ArrayUtils.isEmpty(dim.getDerived())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_DERIVED" : ""));
- newDimensions.add(newDim);
- newDim.setDerived(dim.getDerived());
- newDim.setColumn(null); // derived column must come from a lookup table; in this case the fk will be the dimension column, no need to explicitly declare it;
- needNameSuffix = true;
- }
-
- if (!ArrayUtils.isEmpty(dim.getHierarchy())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_HIERARCHY" : ""));
- newDimensions.add(newDim);
-
- newDim.setHierarchy(true);
-
- List<String> columns = Lists.newArrayList();
- for (HierarchyDesc hierarch : dim.getHierarchy()) {
- String col = hierarch.getColumn();
- columns.add(col);
- }
-
- newDim.setColumn(columns.toArray(new String[columns.size()]));
- }
-
- }
- }
-
- private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- DataModelDesc dm = new DataModelDesc();
- dm.setUuid(UUID.randomUUID().toString());
- String factTable = oldModel.getFactTable();
- dm.setName(oldModel.getName());
- dm.setFactTable(getMetadataManager().appendDBName(factTable));
-
- newModel.setModelName(dm.getName());
-
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<LookupDesc> lookups = Lists.newArrayList();
- List<String> lookupTables = Lists.newArrayList();
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
- JoinDesc join = dim.getJoin();
- if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) {
- String table = dim.getTable();
- table = getMetadataManager().appendDBName(table);
-
- if (lookupTables.contains(table)) {
- continue;
- }
- LookupDesc lookup = new LookupDesc();
- lookup.setJoin(join);
- lookup.setTable(table);
- lookupTables.add(table);
- lookups.add(lookup);
- }
- }
-
- dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
- dm.setFilterCondition(oldModel.getFilterCondition());
- updatePartitionDesc(oldModel, dm);
-
- if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE);
- }
-
- return dm;
- }
-
- private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) {
-
- CubePartitionDesc partition = oldModel.getCubePartitionDesc();
- PartitionDesc newPartition = new PartitionDesc();
-
- if (partition.getPartitionDateColumn() != null) {
- String partitionCol = partition.getPartitionDateColumn();
-
- String[] tablecolumn = partitionCol.split("\\.");
- if (tablecolumn != null && tablecolumn.length == 2) {
- // pattern is <tablename>.<colname>
- String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]);
- newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]);
- } else {
-
- if (partitionCol.indexOf(".") < 0) {
- // pattern is <colname>
- partitionCol = dm.getFactTable() + "." + partitionCol;
- }
-
- newPartition.setPartitionDateColumn(partitionCol);
- }
- }
-
- // only append is supported
- newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND);
-
- newPartition.setPartitionDateStart(partition.getPartitionDateStart());
-
- dm.setPartitionDesc(newPartition);
- }
-
- private CubeDesc loadOldCubeDesc(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("CubeDesc name must not be blank");
- }
-
- Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap();
- Map<String, TableDesc> newMap = Maps.newHashMap();
- for (Entry<String, TableDesc> entry : tableMap.entrySet()) {
- String t = entry.getKey();
-
- if (t.indexOf(".") > 0) {
- newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue());
-
- }
- }
- ndesc.init(KylinConfig.getInstanceFromEnv(), newMap);
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
- }
-
- return ndesc;
- }
-
- private static MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
-
- protected static ResourceStore getStore() {
- return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
deleted file mode 100644
index 8d771b9..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.Collection;
-import java.util.HashSet;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
- *
- * the unified logic for defining a sql's dimension
- */
-public class CubeDimensionDeriver {
-
- public static Collection<TblColRef> getDimensionColumns(Collection<TblColRef> groupByColumns, Collection<TblColRef> filterColumns) {
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
deleted file mode 100644
index 7773351..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeInstance extends RootPersistentEntity implements IRealization {
-
- public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
- CubeInstance cubeInstance = new CubeInstance();
-
- cubeInstance.setConfig(cubeDesc.getConfig());
- cubeInstance.setName(cubeName);
- cubeInstance.setDescName(cubeDesc.getName());
- cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
- cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
- cubeInstance.updateRandomUuid();
- cubeInstance.setProjectName(projectName);
-
- return cubeInstance;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private RealizationStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<CubeSegment> segments = new ArrayList<CubeSegment>();
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- private String projectName;
-
- public List<CubeSegment> getBuildingSegments() {
- List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
- if (null != segments) {
- for (CubeSegment segment : segments) {
- if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
- buildingSegments.add(segment);
- }
- }
- }
-
- return buildingSegments;
- }
-
- public List<CubeSegment> getMergingSegments(CubeSegment mergedSegment) {
- List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>();
- if (null != this.segments) {
- for (CubeSegment segment : this.segments) {
- if (!mergedSegment.equals(segment) //
- && mergedSegment.getDateRangeStart() <= segment.getDateRangeStart() && mergedSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) {
- mergingSegments.add(segment);
- }
- }
- }
- return mergingSegments;
- }
-
- public CubeDesc getDescriptor() {
- return CubeDescManager.getInstance(config).getCubeDesc(descName);
- }
-
- public boolean isReady() {
- return getStatus() == RealizationStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return getCanonicalName();
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("input_records_count")
- public long getInputRecordCount() {
- long sizeRecordCount = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getInputRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("input_records_size")
- public long getInputRecordSize() {
- long sizeRecordSize = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getCanonicalName() {
- return getType() + "[name=" + name + "]";
- }
-
- @Override
- public String getFactTable() {
- return this.getDescriptor().getFactTable();
- }
-
- @Override
- public List<MeasureDesc> getMeasures() {
- return getDescriptor().getMeasures();
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName.toUpperCase();
- }
-
- public String getOriginDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public void setCost(int cost) {
- this.cost = cost;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(RealizationStatusEnum status) {
- this.status = status;
- }
-
- public CubeSegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public CubeSegment getLatestReadySegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getStatus() != SegmentStatusEnum.READY)
- continue;
- if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
- latest = seg;
- }
- }
- return latest;
- }
-
- public List<CubeSegment> getSegments() {
- return segments;
- }
-
- public List<CubeSegment> getSegments(SegmentStatusEnum status) {
- List<CubeSegment> result = new ArrayList<CubeSegment>();
-
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public List<CubeSegment> getSegment(SegmentStatusEnum status) {
- List<CubeSegment> result = Lists.newArrayList();
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
- return result;
- }
-
- public CubeSegment getSegment(String name, SegmentStatusEnum status) {
- for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<CubeSegment> segments) {
- this.segments = segments;
- }
-
- public CubeSegment getSegmentById(String segmentId) {
- for (CubeSegment segment : segments) {
- if (Objects.equal(segment.getUuid(), segmentId)) {
- return segment;
- }
- }
- return null;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- @Override
- public CapabilityResult isCapable(SQLDigest digest) {
- CapabilityResult result = CubeCapabilityChecker.check(this, digest);
- if (result.capable) {
- result.cost = getCost(digest);
- for (CapabilityResult.CapabilityInfluence i : result.influences) {
- result.cost *= (i.suggestCostMultiplier() == 0) ? 1.0 : i.suggestCostMultiplier();
- }
- } else {
- result.cost = -1;
- }
- return result;
- }
-
- private int getCost(SQLDigest digest) {
- return cost;
- }
-
- @Override
- public RealizationType getType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return Lists.newArrayList(getDescriptor().listAllColumns());
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
-
- public boolean needAutoMerge() {
- if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
- return false;
-
- return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
- }
-
-
- @Override
- public long getDateRangeStart() {
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeStart() < startTime)
- startTime = seg.getDateRangeStart();
- }
-
- return startTime;
- }
-
- @Override
- public long getDateRangeEnd() {
-
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = 0;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeEnd() > endTime)
- endTime = seg.getDateRangeEnd();
- }
-
- return endTime;
- }
-
- @Override
- public List<TblColRef> getAllDimensions() {
- return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
deleted file mode 100644
index d3bbf59..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ /dev/null
@@ -1,750 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.IRealizationProvider;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-
-/**
- * @author yangli9
- */
-public class CubeManager implements IRealizationProvider {
-
- private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
- private static int HBASE_TABLE_LENGTH = 10;
- public static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class);
-
- private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>();
-
- public static CubeManager getInstance(KylinConfig config) {
- CubeManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one cubemanager singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // cube name ==> CubeInstance
- private CaseInsensitiveStringCache<CubeInstance> cubeMap = new CaseInsensitiveStringCache<CubeInstance>(Broadcaster.TYPE.CUBE);
- // "table/column" ==> lookup table
- // private SingleValueCache<String, LookupStringTable> lookupTables = new SingleValueCache<String, LookupStringTable>(Broadcaster.TYPE.METADATA);
-
- // for generation hbase table name of a new segment
- private Multimap<String, String> usedStorageLocation = HashMultimap.create();
-
- private CubeManager(KylinConfig config) throws IOException {
- logger.info("Initializing CubeManager with config " + config);
- this.config = config;
-
- loadAllCubeInstance();
- }
-
- public List<CubeInstance> listAllCubes() {
- return new ArrayList<CubeInstance>(cubeMap.values());
- }
-
- public CubeInstance getCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- return cubeMap.get(cubeName);
- }
-
- /**
- * Get related Cubes by cubedesc name. By default, the desc name will be
- * translated into upper case.
- *
- * @param descName CubeDesc name
- * @return
- */
- public List<CubeInstance> getCubesByDesc(String descName) {
-
- descName = descName.toUpperCase();
- List<CubeInstance> list = listAllCubes();
- List<CubeInstance> result = new ArrayList<CubeInstance>();
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- CubeInstance ci = it.next();
- if (descName.equals(ci.getDescName())) {
- result.add(ci);
- }
- }
- return result;
- }
-
- public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- if (!cubeDesc.getRowkey().isUseDictionary(col))
- return null;
-
- DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
-
- if (dictInfo != null) {
- cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
- }
-
- return dictInfo;
- }
-
- /**
- * return null if no dictionary for given column
- */
- public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) {
- DictionaryInfo info = null;
- try {
- DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
- String dictResPath = cubeSeg.getDictResPath(col);
- if (dictResPath == null)
- return null;
-
- info = dictMgr.getDictionaryInfo(dictResPath);
- if (info == null)
- throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid cube state; cube segment" + cubeSeg + ", col " + col);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
- }
-
- return (Dictionary<String>) info.getDictionaryObject();
- }
-
- public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
- SnapshotManager snapshotMgr = getSnapshotManager();
-
- HiveTable hiveTable = new HiveTable(metaMgr, lookupTable);
- TableDesc tableDesc = metaMgr.getTableDesc(lookupTable);
- SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);
-
- cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
-
- saveResource(cubeSeg.getCubeInstance());
-
- return snapshot;
- }
-
- // sync on update
- public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException {
- logger.info("Dropping cube '" + cubeName + "'");
- // load projects before remove cube from project
-
- ResourceStore store = getStore();
-
- // delete cube instance and cube desc
- CubeInstance cube = getCube(cubeName);
-
- if (deleteDesc && cube.getDescriptor() != null)
- store.deleteResource(cube.getDescriptor().getResourcePath());
-
- store.deleteResource(cube.getResourcePath());
-
- // delete cube from project
- ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
-
- // clean cube cache
- this.afterCubeDropped(cube);
-
- return cube;
- }
-
- // sync on update
- public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
- logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
-
- // save cube resource
- CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
- cube.setOwner(owner);
- saveResource(cube);
-
- ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
-
- return cube;
- }
-
- public CubeInstance updateCube(CubeInstance cube) throws IOException {
- logger.info("Updating cube instance '" + cube.getName());
- saveResource(cube);
- return cube;
- }
-
- public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- if (cube.getSegments().size() == 0)
- throw new IllegalStateException("expect at least one existing segment");
-
- long appendStart = calculateStartDateForAppendSegment(cube);
- CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
-
- long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
-
- validateNewSegments(cube, mergeSegment);
- cube.getSegments().add(appendSegment);
- cube.getSegments().add(mergeSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
- }
-
- public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment;
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
- long startDate = calculateStartDateForAppendSegment(cube);
- newSegment = newSegment(cube, startDate, endDate);
- } else {
- newSegment = newSegment(cube, 0, Long.MAX_VALUE);
- }
-
- validateNewSegments(cube, newSegment);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment = newSegment(cube, startDate, endDate);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
- CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
- List<CubeSegment> mergingSegments = cube.getMergingSegments(newSegment);
-
- if (forceMergeEmptySeg == false) {
- List<String> emptySegment = Lists.newArrayList();
- for (CubeSegment seg : mergingSegments) {
- if (seg.getSizeKB() == 0) {
- emptySegment.add(seg.getName());
- }
- }
- if (emptySegment.size() > 0) {
- throw new IllegalArgumentException("Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment);
- }
- }
-
- validateNewSegments(cube, newSegment);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- private Pair<Long, Long> alignMergeRange(CubeInstance cube, long startDate, long endDate) {
- List<CubeSegment> readySegments = cube.getSegment(SegmentStatusEnum.READY);
- if (readySegments.isEmpty()) {
- throw new IllegalStateException("there are no segments in ready state");
- }
- long start = Long.MAX_VALUE;
- long end = Long.MIN_VALUE;
- for (CubeSegment readySegment : readySegments) {
- if (hasOverlap(startDate, endDate, readySegment.getDateRangeStart(), readySegment.getDateRangeEnd())) {
- if (start > readySegment.getDateRangeStart()) {
- start = readySegment.getDateRangeStart();
- }
- if (end < readySegment.getDateRangeEnd()) {
- end = readySegment.getDateRangeEnd();
- }
- }
- }
- return new Pair<Long, Long>(start, end);
- }
-
- private boolean hasOverlap(long startDate, long endDate, long anotherStartDate, long anotherEndDate) {
- if (startDate >= endDate) {
- throw new IllegalArgumentException("startDate must be less than endDate");
- }
- if (anotherStartDate >= anotherEndDate) {
- throw new IllegalArgumentException("anotherStartDate must be less than anotherEndDate");
- }
- if (startDate <= anotherStartDate && anotherStartDate < endDate) {
- return true;
- }
- if (startDate < anotherEndDate && anotherEndDate <= endDate) {
- return true;
- }
- return false;
- }
-
- private long calculateStartDateForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- } else {
- return existing.get(existing.size() - 1).getDateRangeEnd();
- }
- }
-
- private void checkNoBuildingSegment(CubeInstance cube) {
- if (cube.getBuildingSegments().size() > 0) {
- throw new IllegalStateException("There is already a building segment!");
- }
- }
-
- private void checkCubeIsPartitioned(CubeInstance cube) {
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) {
- throw new IllegalStateException("there is no partition date column specified, only full build is supported");
- }
- }
-
- public void updateSegmentOnJobDiscard(CubeInstance cubeInstance, String segmentName) throws IOException {
- for (int i = 0; i < cubeInstance.getSegments().size(); i++) {
- CubeSegment segment = cubeInstance.getSegments().get(i);
- if (segment.getName().equals(segmentName) && segment.getStatus() != SegmentStatusEnum.READY) {
- cubeInstance.getSegments().remove(segment);
- }
- }
- updateCube(cubeInstance);
- }
-
- /**
- * After cube update, reload cube related cache
- *
- * @param cubeName
- */
- public void loadCubeCache(String cubeName) {
- try {
- loadCubeInstance(CubeInstance.concatResourcePath(cubeName));
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
-
- /**
- * After cube deletion, remove cube related cache
- *
- * @param cube
- */
- public void removeCubeCache(CubeInstance cube) {
- final String cubeName = cube.getName().toUpperCase();
- cubeMap.remove(cubeName);
- usedStorageLocation.removeAll(cubeName);
- Cuboid.reloadCache(cube.getDescName());
- }
-
- public void removeCubeCacheLocal(String cubeName) {
- cubeMap.removeLocal(cubeName);
- usedStorageLocation.removeAll(cubeName);
- }
-
- public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
-
- String tableName = dim.getTable();
- String[] pkCols = dim.getJoin().getPrimaryKey();
- String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
- if (snapshotResPath == null)
- throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
-
- try {
- SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
- return new LookupStringTable(tableDesc, pkCols, snapshot);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
- }
- }
-
- private void saveResource(CubeInstance cube) throws IOException {
- ResourceStore store = getStore();
- store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- this.afterCubeUpdated(cube);
- }
-
- private void afterCubeUpdated(CubeInstance updatedCube) {
- cubeMap.put(updatedCube.getName(), updatedCube);
- }
-
- private void afterCubeDropped(CubeInstance droppedCube) {
- removeCubeCache(droppedCube);
- }
-
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
- public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
- List<CubeSegment> tobe = calculateToBeSegments(cube);
-
- for (CubeSegment seg : newSegments) {
- if (tobe.contains(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " is expected but not in the tobe " + tobe);
-
- if (StringUtils.isBlank(seg.getStorageLocationIdentifier()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing StorageLocationIdentifier");
-
- if (StringUtils.isBlank(seg.getLastBuildJobID()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing LastBuildJobID");
-
- seg.setStatus(SegmentStatusEnum.READY);
- }
-
- for (CubeSegment seg : tobe) {
- if (isReady(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " should be READY but is not");
- }
-
- cube.setSegments(tobe);
- cube.setStatus(RealizationStatusEnum.READY);
-
- logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- saveResource(cube);
- }
-
- private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
- List<CubeSegment> tobe = calculateToBeSegments(cube, newSegments);
- List<CubeSegment> newList = Arrays.asList(newSegments);
- if (tobe.containsAll(newList) == false) {
- throw new IllegalStateException("For cube " + cube + ", the new segments " + newList + " do not fit in its current " + cube.getSegments() + "; the resulted tobe is " + tobe);
- }
- }
-
- /**
- * Smartly figure out the TOBE segments once all new segments are built.
- * - Ensures no gap, no overlap
- * - Favors new segments over the old
- * - Favors big segments over the small
- */
- private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment... newSegments) {
- CubeDesc cubeDesc = cube.getDescriptor();
- PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc();
-
- List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
- if (newSegments != null)
- tobe.addAll(Arrays.asList(newSegments));
- if (tobe.size() == 0)
- return tobe;
-
- // sort by start time, then end time
- Collections.sort(tobe);
-
- // check first segment start time
- CubeSegment firstSeg = tobe.get(0);
- firstSeg.validate();
-
- for (int i = 0, j = 1; j < tobe.size();) {
- CubeSegment is = tobe.get(i);
- CubeSegment js = tobe.get(j);
- js.validate();
-
- // check i is either ready or new
- if (!isNew(is) && !isReady(is)) {
- tobe.remove(i);
- continue;
- }
-
- // check j is either ready or new
- if (!isNew(js) && !isReady(js)) {
- tobe.remove(j);
- continue;
- }
-
- // if i, j competes
- if (is.getDateRangeStart() == js.getDateRangeStart()) {
- // if both new or ready, favor the bigger segment
- if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
- if (is.getDateRangeEnd() <= js.getDateRangeEnd()) {
- tobe.remove(i);
- } else {
- tobe.remove(j);
- }
- }
- // otherwise, favor the new segment
- else if (isNew(is)) {
- tobe.remove(j);
- } else {
- tobe.remove(i);
- }
- continue;
- }
-
- // if i, j in sequence
- if (is.getDateRangeEnd() == js.getDateRangeStart()) {
- i++;
- j++;
- continue;
- }
-
- // seems j not fitting
- tobe.remove(j);
- }
-
- return tobe;
- }
-
- private boolean isReady(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.READY;
- }
-
- private boolean isNew(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
- }
-
- private void loadAllCubeInstance() throws IOException {
- ResourceStore store = getStore();
- List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");
-
- logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
-
- for (String path : paths) {
- loadCubeInstance(path);
- }
-
- logger.debug("Loaded " + paths.size() + " Cube(s)");
- }
-
- private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeInstance cubeInstance;
- try {
- cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
- cubeInstance.setConfig(config);
-
- if (StringUtils.isBlank(cubeInstance.getName()))
- throw new IllegalStateException("CubeInstance name must not be blank, at " + path);
-
- if (cubeInstance.getDescriptor() == null)
- throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path);
-
- final String cubeName = cubeInstance.getName();
- cubeMap.putLocal(cubeName, cubeInstance);
-
- for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
- }
-
- return cubeInstance;
- } catch (Exception e) {
- logger.error("Error during load cube instance " + path, e);
- return null;
- }
- }
-
- public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
- if (!cube.needAutoMerge()) {
- logger.debug("Cube " + cube.getName() + " doesn't need auto merge");
- return null;
- }
-
- if (cube.getBuildingSegments().size() > 0) {
- logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment");
- return null;
- }
-
- List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegment(SegmentStatusEnum.READY));
-
- if (readySegments.size() == 0) {
- logger.debug("Cube " + cube.getName() + " has no ready segment to merge");
- return null;
- }
-
- long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
- Arrays.sort(timeRanges);
-
- CubeSegment newSeg = null;
- for (int i = timeRanges.length - 1; i >= 0; i--) {
- long toMergeRange = timeRanges[i];
- long currentRange = 0;
- long lastEndTime = 0;
- List<CubeSegment> toMergeSegments = Lists.newArrayList();
- for (CubeSegment segment : readySegments) {
- long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart();
-
- if (thisSegmentRange >= toMergeRange) {
- // this segment and its previous segments will not be merged
- toMergeSegments.clear();
- currentRange = 0;
- lastEndTime = segment.getDateRangeEnd();
- continue;
- }
-
- if (segment.getDateRangeStart() != lastEndTime && toMergeSegments.isEmpty() == false) {
- // gap exists, give up the small segments before the gap;
- toMergeSegments.clear();
- currentRange = 0;
- }
-
- currentRange += thisSegmentRange;
- if (currentRange < toMergeRange) {
- toMergeSegments.add(segment);
- lastEndTime = segment.getDateRangeEnd();
- } else {
- // merge
- toMergeSegments.add(segment);
-
- newSeg = newSegment(cube, toMergeSegments.get(0).getDateRangeStart(), segment.getDateRangeEnd());
- // only one merge job be created here
- return newSeg;
- }
- }
-
- }
-
- return null;
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private DictionaryManager getDictionaryManager() {
- return DictionaryManager.getInstance(config);
- }
-
- private SnapshotManager getSnapshotManager() {
- return SnapshotManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- @Override
- public RealizationType getRealizationType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public IRealization getRealization(String name) {
- return getCube(name);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
deleted file mode 100644
index 12f0bf7..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, ISegment {
-
- @JsonBackReference
- private CubeInstance cubeInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private SegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("input_records")
- private long inputRecords;
- @JsonProperty("input_records_size")
- private long inputRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of cube schema and dictionary ID, used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path
- @JsonProperty("snapshots")
- private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
-
- public CubeDesc getCubeDesc() {
- return getCubeInstance().getDescriptor();
- }
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public SegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(SegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getInputRecords() {
- return inputRecords;
- }
-
- public void setInputRecords(long inputRecords) {
- this.inputRecords = inputRecords;
- }
-
- public long getInputRecordsSize() {
- return inputRecordsSize;
- }
-
- public void setInputRecordsSize(long inputRecordsSize) {
- this.inputRecordsSize = inputRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public CubeInstance getCubeInstance() {
- return cubeInstance;
- }
-
- public void setCubeInstance(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- }
-
- public String getStorageLocationIdentifier() {
-
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Map<String, String> getSnapshots() {
- if (snapshots == null)
- snapshots = new ConcurrentHashMap<String, String>();
- return snapshots;
- }
-
- public String getSnapshotResPath(String table) {
- return getSnapshots().get(table);
- }
-
- public void putSnapshotResPath(String table, String snapshotResPath) {
- getSnapshots().put(table, snapshotResPath);
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public Collection<String> getSnapshotPaths() {
- return getSnapshots().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
- Dictionary<?> dict = getDictionary(col);
- if (dict == null) {
- return this.getCubeDesc().getRowkey().getColumnLength(col);
- } else {
- return dict.getSizeOfId();
- }
- }
-
- @Override
- public Dictionary<String> getDictionary(TblColRef col) {
- return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
- }
-
- public Map<TblColRef, Dictionary<String>> buildDictionaryMap() {
- Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
- for (TblColRef col : getCubeDesc().getAllColumnsNeedDictionary()) {
- result.put(col, (Dictionary<String>) getDictionary(col));
- }
- return result;
- }
-
-
- public void validate() {
- if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned() && dateRangeStart >= dateRangeEnd)
- throw new IllegalStateException("dateRangeStart(" + dateRangeStart + ") must be greater than dateRangeEnd(" + dateRangeEnd + ") in segment " + this);
- }
-
- @Override
- public int compareTo(CubeSegment other) {
- long comp = this.dateRangeStart - other.dateRangeStart;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
-
- comp = this.dateRangeEnd - other.dateRangeEnd;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
- else
- return 0;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((status == null) ? 0 : status.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CubeSegment other = (CubeSegment) obj;
- if (cubeInstance == null) {
- if (other.cubeInstance != null)
- return false;
- } else if (!cubeInstance.equals(other.cubeInstance))
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (status != other.status)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-
- public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {
- this.dictionaries = dictionaries;
- }
-
- public void setSnapshots(ConcurrentHashMap<String, String> snapshots) {
- this.snapshots = snapshots;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
deleted file mode 100644
index 3cdaa93..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cli;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DictionaryGeneratorCLI {
-
- private static final Logger logger = LoggerFactory.getLogger(DictionaryGeneratorCLI.class);
-
- public static void processSegment(KylinConfig config, String cubeName, String segmentName, String factColumnsPath) throws IOException {
- CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
- CubeSegment segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
-
- processSegment(config, segment, factColumnsPath);
- }
-
- private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
- CubeManager cubeMgr = CubeManager.getInstance(config);
-
- for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- logger.info("Building dictionary for " + col);
- cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
- }
- }
-
- // build snapshot
- if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
- // CubeSegment seg = cube.getTheOnlySegment();
- logger.info("Building snapshot of " + dim.getTable());
- cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
- logger.info("Checking snapshot of " + dim.getTable());
- cubeMgr.getLookupTable(cubeSeg, dim); // load the table for
- // sanity check
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
deleted file mode 100644
index d4613e9..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cli;
-
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Date;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryInfoSerializer;
-
-public class DumpDictionaryCLI {
-
- public static void main(String[] args) throws IOException {
- for (String path : args) {
- dump(new File(path));
- }
- }
-
- public static void dump(File f) throws IOException {
- if (f.isDirectory()) {
- for (File c : f.listFiles())
- dump(c);
- return;
- }
-
- if (f.getName().endsWith(".dict")) {
- DictionaryInfoSerializer ser = new DictionaryInfoSerializer();
- DictionaryInfo dictInfo = ser.deserialize(new DataInputStream(new FileInputStream(f)));
-
- System.out.println("============================================================================");
- System.out.println("File: " + f.getAbsolutePath());
- System.out.println(new Date(dictInfo.getLastModified()));
- System.out.println(JsonUtil.writeValueAsIndentString(dictInfo));
- dictInfo.getDictionaryObject().dump(System.out);
- System.out.println();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
deleted file mode 100644
index 4821c94..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.common;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeySplitter {
-
- private CubeDesc cubeDesc;
- private RowKeyColumnIO colIO;
-
- private SplittedBytes[] splitBuffers;
- private int bufferSize;
-
- public SplittedBytes[] getSplitBuffers() {
- return splitBuffers;
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
- this.cubeDesc = cubeSeg.getCubeDesc();
- this.colIO = new RowKeyColumnIO(cubeSeg);
-
- this.splitBuffers = new SplittedBytes[splitLen];
- for (int i = 0; i < splitLen; i++) {
- this.splitBuffers[i] = new SplittedBytes(bytesLen);
- }
- this.bufferSize = 0;
- }
-
- /**
- * @param bytes
- * @param byteLen
- * @return cuboid ID
- */
- public long split(byte[] bytes, int byteLen) {
- this.bufferSize = 0;
- int offset = 0;
-
- // extract cuboid id
- SplittedBytes cuboidIdSplit = this.splitBuffers[this.bufferSize++];
- cuboidIdSplit.length = RowConstants.ROWKEY_CUBOIDID_LEN;
- System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
- offset += RowConstants.ROWKEY_CUBOIDID_LEN;
-
- long cuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
- Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
-
- // rowkey columns
- for (int i = 0; i < cuboid.getColumns().size(); i++) {
- TblColRef col = cuboid.getColumns().get(i);
- int colLength = colIO.getColumnLength(col);
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- split.length = colLength;
- System.arraycopy(bytes, offset, split.value, 0, colLength);
- offset += colLength;
- }
-
- return cuboidId;
- }
-}
[51/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
KYLIN-1416 keep only website in document branch
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b6aa313
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b6aa313
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b6aa313
Branch: refs/heads/document
Commit: 6b6aa3135e32ea828e37de23ddc1d1f88703e17b
Parents: fc66cdb
Author: Yang Li <li...@apache.org>
Authored: Thu Feb 11 20:48:43 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Feb 11 20:48:43 2016 +0800
----------------------------------------------------------------------
.gitignore | 105 -
.reviewboardrc | 4 -
.settings/org.eclipse.core.resources.prefs | 2 -
KEYS | 131 -
LICENSE | 247 -
NOTICE | 6 -
README.md | 26 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
.../.settings/org.eclipse.jdt.core.prefs | 379 -
atopcalcite/.settings/org.eclipse.jdt.ui.prefs | 7 -
atopcalcite/pom.xml | 47 -
.../apache/calcite/sql2rel/RelFieldTrimmer.java | 44 -
.../calcite/sql2rel/SqlToRelConverter.java | 5028 -
bin/check-env.sh | 55 -
bin/find-hbase-dependency.sh | 41 -
bin/find-hive-dependency.sh | 70 -
bin/get-properties.sh | 32 -
bin/health-check.sh | 50 -
bin/kylin.sh | 135 -
bin/metastore.sh | 67 -
bin/performance-monitor.sh | 38 -
bin/sample.sh | 30 -
bin/setenv.sh | 54 -
.../.settings/org.eclipse.core.resources.prefs | 5 -
common/.settings/org.eclipse.jdt.core.prefs | 379 -
common/.settings/org.eclipse.jdt.ui.prefs | 7 -
common/pom.xml | 137 -
.../org/apache/kylin/common/KylinConfig.java | 775 -
.../kylin/common/debug/BackdoorToggles.java | 98 -
.../common/hll/HyperLogLogPlusCounter.java | 340 -
.../kylin/common/hll/HyperLogLogPlusTable.java | 199 -
.../org/apache/kylin/common/lock/JobLock.java | 26 -
.../apache/kylin/common/lock/MockJobLock.java | 32 -
.../kylin/common/lock/ZookeeperJobLock.java | 100 -
.../org/apache/kylin/common/mr/KylinMapper.java | 32 -
.../apache/kylin/common/mr/KylinReducer.java | 32 -
.../kylin/common/persistence/AclEntity.java | 29 -
.../common/persistence/FileResourceStore.java | 155 -
.../common/persistence/HBaseConnection.java | 122 -
.../common/persistence/HBaseResourceStore.java | 345 -
.../common/persistence/JsonSerializer.java | 47 -
.../kylin/common/persistence/RawResource.java | 34 -
.../kylin/common/persistence/ResourceStore.java | 290 -
.../kylin/common/persistence/ResourceTool.java | 135 -
.../persistence/RootPersistentEntity.java | 124 -
.../kylin/common/persistence/Serializer.java | 35 -
.../common/persistence/StorageException.java | 38 -
.../common/restclient/AbstractRestCache.java | 48 -
.../kylin/common/restclient/Broadcaster.java | 233 -
.../restclient/CaseInsensitiveStringCache.java | 41 -
.../common/restclient/MultiValueCache.java | 88 -
.../kylin/common/restclient/RestClient.java | 129 -
.../common/restclient/SingleValueCache.java | 96 -
.../org/apache/kylin/common/util/Array.java | 71 -
.../org/apache/kylin/common/util/ByteArray.java | 57 -
.../org/apache/kylin/common/util/Bytes.java | 2203 -
.../kylin/common/util/BytesSerializer.java | 35 -
.../apache/kylin/common/util/BytesSplitter.java | 162 -
.../org/apache/kylin/common/util/BytesUtil.java | 423 -
.../common/util/CaseInsensitiveStringMap.java | 105 -
.../org/apache/kylin/common/util/ClassUtil.java | 78 -
.../kylin/common/util/CliCommandExecutor.java | 147 -
.../apache/kylin/common/util/DateFormat.java | 105 -
.../apache/kylin/common/util/Dictionary.java | 199 -
.../common/util/HBaseRegionSizeCalculator.java | 127 -
.../apache/kylin/common/util/HadoopUtil.java | 153 -
.../apache/kylin/common/util/HiveClient.java | 172 -
.../org/apache/kylin/common/util/JsonUtil.java | 86 -
.../kylin/common/util/Log4jConfigurer.java | 55 -
.../kylin/common/util/LogTitlePrinter.java | 32 -
.../org/apache/kylin/common/util/Logger.java | 27 -
.../apache/kylin/common/util/MailService.java | 102 -
.../kylin/common/util/MyLogFormatter.java | 78 -
.../java/org/apache/kylin/common/util/Pair.java | 117 -
.../apache/kylin/common/util/PartialSorter.java | 43 -
.../apache/kylin/common/util/RandomSampler.java | 55 -
.../org/apache/kylin/common/util/SSHClient.java | 378 -
.../kylin/common/util/SSHClientOutput.java | 53 -
.../apache/kylin/common/util/SoutLogger.java | 29 -
.../apache/kylin/common/util/SplittedBytes.java | 33 -
.../kylin/common/util/StringSplitter.java | 47 -
.../apache/kylin/common/util/StringUtil.java | 109 -
.../org/apache/kylin/common/util/TarGZUtil.java | 69 -
.../apache/kylin/common/util/ThreadUtil.java | 48 -
.../src/main/resources/kylin-log4j.properties | 28 -
.../persistence/HBaseResourceStoreTest.java | 211 -
.../persistence/LocalFileResourceStoreTest.java | 171 -
.../common/persistence/ResourceToolTest.java | 45 -
.../kylin/common/restclient/RestClientTest.java | 41 -
.../common/util/AbstractKylinTestCase.java | 85 -
.../kylin/common/util/BasicHadoopTest.java | 83 -
.../org/apache/kylin/common/util/BasicTest.java | 88 -
.../apache/kylin/common/util/BytesUtilTest.java | 65 -
.../common/util/HBaseMetadataTestCase.java | 74 -
.../common/util/HBaseMiniclusterHelper.java | 166 -
.../kylin/common/util/HadoopUtilTest.java | 65 -
.../apache/kylin/common/util/HbaseImporter.java | 119 -
.../common/util/HyperLogLogCounterTest.java | 219 -
.../common/util/InstallJarIntoMavenTest.java | 63 -
.../common/util/LocalFileMetadataTestCase.java | 73 -
.../kylin/common/util/MailServiceTest.java | 74 -
.../kylin/common/util/PartialSorterTest.java | 51 -
.../kylin/common/util/RandomSamplerTest.java | 48 -
.../apache/kylin/common/util/SSHClientTest.java | 84 -
conf/kylin.properties | 149 -
conf/kylin_job_conf.xml | 93 -
cube/.settings/org.eclipse.core.resources.prefs | 6 -
cube/.settings/org.eclipse.jdt.core.prefs | 379 -
cube/.settings/org.eclipse.jdt.ui.prefs | 7 -
cube/pom.xml | 190 -
cube/src/main/assembly/assemble.xml | 33 -
.../kylin/cube/CubeCapabilityChecker.java | 196 -
.../org/apache/kylin/cube/CubeDescManager.java | 280 -
.../org/apache/kylin/cube/CubeDescUpgrader.java | 308 -
.../apache/kylin/cube/CubeDimensionDeriver.java | 39 -
.../org/apache/kylin/cube/CubeInstance.java | 400 -
.../java/org/apache/kylin/cube/CubeManager.java | 750 -
.../java/org/apache/kylin/cube/CubeSegment.java | 348 -
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 68 -
.../kylin/cube/cli/DumpDictionaryCLI.java | 58 -
.../kylin/cube/common/RowKeySplitter.java | 91 -
.../org/apache/kylin/cube/cuboid/Cuboid.java | 352 -
.../org/apache/kylin/cube/cuboid/CuboidCLI.java | 170 -
.../kylin/cube/cuboid/CuboidScheduler.java | 194 -
.../cube/estimation/CubeSizeEstimationCLI.java | 181 -
.../kylin/cube/kv/AbstractRowKeyEncoder.java | 59 -
.../apache/kylin/cube/kv/FuzzyKeyEncoder.java | 43 -
.../apache/kylin/cube/kv/FuzzyMaskEncoder.java | 55 -
.../org/apache/kylin/cube/kv/RowConstants.java | 51 -
.../apache/kylin/cube/kv/RowKeyColumnIO.java | 150 -
.../apache/kylin/cube/kv/RowKeyColumnOrder.java | 108 -
.../org/apache/kylin/cube/kv/RowKeyDecoder.java | 140 -
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 142 -
.../apache/kylin/cube/kv/RowValueDecoder.java | 139 -
.../kylin/cube/model/CubeBuildTypeEnum.java | 39 -
.../org/apache/kylin/cube/model/CubeDesc.java | 830 -
.../apache/kylin/cube/model/DimensionDesc.java | 241 -
.../kylin/cube/model/HBaseColumnDesc.java | 123 -
.../kylin/cube/model/HBaseColumnFamilyDesc.java | 60 -
.../kylin/cube/model/HBaseMappingDesc.java | 98 -
.../apache/kylin/cube/model/HierarchyDesc.java | 70 -
.../apache/kylin/cube/model/RowKeyColDesc.java | 92 -
.../org/apache/kylin/cube/model/RowKeyDesc.java | 288 -
.../kylin/cube/model/v1/CubeBuildTypeEnum.java | 34 -
.../apache/kylin/cube/model/v1/CubeDesc.java | 831 -
.../kylin/cube/model/v1/CubeInstance.java | 427 -
.../kylin/cube/model/v1/CubePartitionDesc.java | 94 -
.../apache/kylin/cube/model/v1/CubeSegment.java | 303 -
.../cube/model/v1/CubeSegmentStatusEnum.java | 27 -
.../cube/model/v1/CubeSegmentTypeEnum.java | 27 -
.../kylin/cube/model/v1/CubeStatusEnum.java | 25 -
.../kylin/cube/model/v1/DimensionDesc.java | 216 -
.../kylin/cube/model/v1/ProjectInstance.java | 236 -
.../model/validation/CubeMetadataValidator.java | 75 -
.../cube/model/validation/IValidatorRule.java | 30 -
.../cube/model/validation/ResultLevel.java | 38 -
.../SourceTableMetadataValidator.java | 35 -
.../cube/model/validation/ValidateContext.java | 103 -
.../rule/AggregationGroupSizeRule.java | 66 -
.../model/validation/rule/FunctionRule.java | 159 -
.../rule/IKylinValidationConstants.java | 33 -
.../validation/rule/MandatoryColumnRule.java | 75 -
.../model/validation/rule/RowKeyAttrRule.java | 74 -
.../cube/AggregationGroupSizeRuleTest.java | 101 -
.../org/apache/kylin/cube/CubeDescTest.java | 81 -
.../apache/kylin/cube/CubeManagerCacheTest.java | 77 -
.../org/apache/kylin/cube/CubeManagerTest.java | 90 -
.../org/apache/kylin/cube/CubeSegmentsTest.java | 170 -
.../kylin/cube/DictionaryManagerTest.java | 85 -
.../kylin/cube/MandatoryColumnRuleTest.java | 57 -
.../apache/kylin/cube/RowKeyAttrRuleTest.java | 58 -
.../kylin/cube/common/BytesSplitterTest.java | 54 -
.../kylin/cube/common/RowKeySplitterTest.java | 71 -
.../kylin/cube/cuboid/CombinationTest.java | 94 -
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 285 -
.../apache/kylin/cube/cuboid/CuboidTest.java | 184 -
.../apache/kylin/cube/kv/RowKeyDecoderTest.java | 117 -
.../apache/kylin/cube/kv/RowKeyEncoderTest.java | 147 -
.../kylin/cube/kv/RowValueDecoderTest.java | 87 -
.../kylin/cube/project/ProjectManagerTest.java | 177 -
.../kylin/metadata/MetadataUpgradeTest.java | 116 -
.../metadata/measure/MeasureCodecTest.java | 82 -
cube/src/test/resources/data/TEST1_desc.json | 188 -
cube/src/test/resources/data/TEST2_desc.json | 173 -
cube/src/test/resources/data/TEST3_desc.json | 268 -
deploy/server.xml | 148 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
dictionary/.settings/org.eclipse.jdt.core.prefs | 379 -
dictionary/.settings/org.eclipse.jdt.ui.prefs | 7 -
dictionary/pom.xml | 128 -
.../org/apache/kylin/dict/BytesConverter.java | 26 -
.../apache/kylin/dict/DateStrDictionary.java | 186 -
.../apache/kylin/dict/DictionaryGenerator.java | 176 -
.../org/apache/kylin/dict/DictionaryInfo.java | 157 -
.../kylin/dict/DictionaryInfoSerializer.java | 80 -
.../apache/kylin/dict/DictionaryManager.java | 390 -
.../kylin/dict/IDictionaryValueEnumerator.java | 32 -
.../java/org/apache/kylin/dict/ISegment.java | 36 -
.../dict/ListDictionaryValueEnumerator.java | 50 -
.../dict/MultipleDictionaryValueEnumerator.java | 78 -
.../org/apache/kylin/dict/NumberDictionary.java | 188 -
.../kylin/dict/NumberDictionaryBuilder.java | 48 -
.../apache/kylin/dict/StringBytesConverter.java | 35 -
.../kylin/dict/TableColumnValueEnumerator.java | 75 -
.../org/apache/kylin/dict/TrieDictionary.java | 428 -
.../kylin/dict/TrieDictionaryBuilder.java | 536 -
.../org/apache/kylin/dict/lookup/FileTable.java | 106 -
.../kylin/dict/lookup/FileTableReader.java | 217 -
.../org/apache/kylin/dict/lookup/HiveTable.java | 104 -
.../kylin/dict/lookup/HiveTableReader.java | 155 -
.../kylin/dict/lookup/LookupBytesTable.java | 51 -
.../kylin/dict/lookup/LookupStringTable.java | 45 -
.../apache/kylin/dict/lookup/LookupTable.java | 170 -
.../apache/kylin/dict/lookup/ReadableTable.java | 135 -
.../kylin/dict/lookup/SnapshotManager.java | 170 -
.../apache/kylin/dict/lookup/SnapshotTable.java | 244 -
.../dict/lookup/SnapshotTableSerializer.java | 79 -
.../kylin/dict/DateStrDictionaryTest.java | 104 -
.../apache/kylin/dict/HiveTableReaderTest.java | 50 -
.../org/apache/kylin/dict/LookupTableTest.java | 75 -
.../apache/kylin/dict/NumberDictionaryTest.java | 162 -
.../apache/kylin/dict/SnapshotManagerTest.java | 82 -
.../org/apache/kylin/dict/TableReaderTest.java | 48 -
.../apache/kylin/dict/TrieDictionaryTest.java | 343 -
.../kylin/dict/lookup/LookupTableTest.java | 122 -
dictionary/src/test/resources/dict/DW_SITES | Bin 5020 -> 0 bytes
.../dict/dw_category_grouping_names.dat | 448529 ----------------
.../english-words.80 (scowl-2015.05.18).dic | 139304 -----
docs/README.md | 6 -
examples/.gitignore | 2 -
examples/sample_cube/create_sample_tables.sql | 190 -
.../sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv | 731 -
.../data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv | 144 -
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 10000 -
.../metadata/cube/kylin_sales_cube.json | 13 -
.../cube_desc/kylin_sales_cube_desc.json | 166 -
.../metadata/model_desc/kylin_sales_model.json | 28 -
.../metadata/project/learn_kylin.json | 11 -
.../metadata/table/DEFAULT.KYLIN_CAL_DT.json | 407 -
.../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 151 -
.../metadata/table/DEFAULT.KYLIN_SALES.json | 43 -
.../test_kylin_cube_with_slr_1_new_segment.json | 43 -
.../cube/test_kylin_cube_with_slr_empty.json | 10 -
...est_kylin_cube_with_slr_left_join_empty.json | 11 -
...est_kylin_cube_with_slr_left_join_ready.json | 43 -
.../cube/test_kylin_cube_with_slr_ready.json | 46 -
...st_kylin_cube_with_slr_ready_2_segments.json | 73 -
.../cube/test_kylin_cube_without_slr_empty.json | 10 -
..._kylin_cube_without_slr_left_join_empty.json | 11 -
..._kylin_cube_without_slr_left_join_ready.json | 43 -
..._without_slr_left_join_ready_2_segments.json | 74 -
.../cube/test_kylin_cube_without_slr_ready.json | 43 -
.../test_kylin_cube_with_slr_desc.json | 241 -
...test_kylin_cube_with_slr_left_join_desc.json | 241 -
.../test_kylin_cube_without_slr_desc.json | 287 -
...t_kylin_cube_without_slr_left_join_desc.json | 287 -
.../test_case_data/localmeta/data/.gitignore | 1 -
.../data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv | 144 -
.../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv | 402 -
.../localmeta/data/EDW.TEST_CAL_DT.csv | 731 -
.../localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv | 8 -
.../localmeta/data/EDW.TEST_SITES.csv | 262 -
.../localmeta/data/data_gen_config.json | 31 -
.../localmeta/data/flatten_data_for_ii.csv | 402 -
.../9355165b-06ad-4c04-977c-a251e66e7e98.dict | Bin 2467 -> 0 bytes
.../b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict | Bin 2929 -> 0 bytes
.../38361fbc-b875-4273-b8b4-1b8a26ef8570.dict | Bin 1909 -> 0 bytes
.../895739d6-27e1-4ecc-b798-5851c319ea40.dict | Bin 1487 -> 0 bytes
.../30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict | Bin 695 -> 0 bytes
.../1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict | Bin 1049 -> 0 bytes
.../bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict | Bin 641 -> 0 bytes
.../8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict | Bin 790 -> 0 bytes
.../278d7283-518a-4cd0-b6e1-2573b523bf1f.dict | Bin 1459 -> 0 bytes
.../d1094cfe-3a15-4270-b066-241c543b3fd9.dict | Bin 473 -> 0 bytes
.../f7fe5157-f9a0-4f64-ad36-d9c92b9c9e2a.dict | Bin 1809 -> 0 bytes
.../d5c40465-75e1-40bc-a960-06308f0134a6.dict | Bin 586 -> 0 bytes
.../e11524a1-0fcf-4da1-b12f-c9410610ab8d.dict | Bin 600 -> 0 bytes
.../fea3a18d-3f20-4b8b-a880-7af93e69241b.dict | Bin 1110 -> 0 bytes
.../fe91daba-e89b-4911-ad75-41e1335a35cb.dict | Bin 619 -> 0 bytes
.../7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict | Bin 1700 -> 0 bytes
.../5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict | Bin 535 -> 0 bytes
.../962b5f64-bee1-49ee-a072-af882193b719.dict | Bin 540 -> 0 bytes
.../bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict | Bin 699 -> 0 bytes
.../bfb86010-bf4c-4534-a2b0-59d020aed197.dict | Bin 744 -> 0 bytes
.../244af7a2-7352-4b30-811f-46e637d7a133.dict | Bin 628 -> 0 bytes
.../4ef43390-b07e-4d4c-872a-77c0bd783acb.dict | Bin 2412 -> 0 bytes
.../f363531d-e969-4264-bffd-ac18f8f47220.dict | Bin 4440 -> 0 bytes
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
.../aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict | Bin 518 -> 0 bytes
.../ed0c3451-593c-494c-9019-64f63fcb0b8e.dict | Bin 518 -> 0 bytes
.../16d8185c-ee6b-4f8c-a919-756d9809f937.dict | Bin 2515 -> 0 bytes
.../9bb0b83a-97b2-434f-905f-4d9e04b62018.dict | Bin 2488 -> 0 bytes
.../c12ae49d-9dbe-4a58-b169-19afac317696.dict | Bin 2515 -> 0 bytes
.../eaed91b0-4182-4ee5-a733-1047a622ee29.dict | Bin 2515 -> 0 bytes
.../ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict | Bin 2515 -> 0 bytes
.../270fbfb0-281c-4602-8413-2970a7439c47.dict | Bin 2913 -> 0 bytes
.../7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict | Bin 2940 -> 0 bytes
.../8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict | Bin 2940 -> 0 bytes
.../ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict | Bin 2940 -> 0 bytes
.../eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict | Bin 2940 -> 0 bytes
.../2602386c-debb-4968-8d2f-b52b8215e385.dict | Bin 1841 -> 0 bytes
.../4243889f-bc81-4807-a975-7041bbbf35e7.dict | Bin 1841 -> 0 bytes
.../494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict | Bin 1841 -> 0 bytes
.../8b4b1c06-fb74-486b-a2ad-74420afebcda.dict | Bin 1841 -> 0 bytes
.../ac520edd-f9d2-419f-a7de-587bfb97dc81.dict | Bin 1841 -> 0 bytes
.../0410d2c4-4686-40bc-ba14-170042a2de94.dict | Bin 1498 -> 0 bytes
.../51ff5e6c-22b3-444e-9915-d376a10f20cb.dict | Bin 1498 -> 0 bytes
.../70c8d6ef-f55b-4cdd-8be4-9c2b236cc8e9.dict | Bin 1471 -> 0 bytes
.../a4e57e55-48fc-4f25-a9c8-485deed25925.dict | Bin 1498 -> 0 bytes
.../ac797050-e152-4923-a20a-02282fcf6086.dict | Bin 1471 -> 0 bytes
.../aceae914-4246-4251-a0c2-692fe7a300df.dict | Bin 1498 -> 0 bytes
.../b298089f-9656-4693-b9b2-8fea46f06dd5.dict | Bin 1498 -> 0 bytes
.../c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict | Bin 1498 -> 0 bytes
.../48433f91-0d68-495f-b7f2-295414591275.dict | Bin 480 -> 0 bytes
.../73c9bfe1-6496-4ff8-9467-6cbee2924c16.dict | Bin 552 -> 0 bytes
.../4059cab1-9b99-47ed-a494-751da52a7d58.dict | Bin 1730 -> 0 bytes
.../96b7c577-b209-45b3-a848-4d2d7af5c0cc.dict | Bin 1842 -> 0 bytes
.../9fc2360e-172a-43f3-acef-be16748a9bb7.dict | Bin 1751 -> 0 bytes
.../bd9f6b22-36ba-4e6b-92aa-0d585faf0b39.dict | Bin 593 -> 0 bytes
.../7df5789b-0280-453c-b406-b75cad6770d1.dict | Bin 613 -> 0 bytes
.../927dde3f-6999-4434-b57c-adfa73160334.dict | Bin 1223 -> 0 bytes
.../6e285e1e-ed16-4012-9f1e-f950dd6927ce.dict | Bin 1117 -> 0 bytes
.../8300bf83-053e-48bb-8c87-88c8d483afd1.dict | Bin 586 -> 0 bytes
.../c8d19f95-b6cd-4219-a114-54aaddcb2909.dict | Bin 1707 -> 0 bytes
.../0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict | Bin 661 -> 0 bytes
.../14fe66b3-5956-498c-bd93-40182cac5510.dict | Bin 661 -> 0 bytes
.../1d383a36-81b9-4177-a822-04eab3683e5b.dict | Bin 661 -> 0 bytes
.../2a44ff38-f64b-42e7-9fcf-66afccac8047.dict | Bin 634 -> 0 bytes
.../c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict | Bin 661 -> 0 bytes
.../0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict | Bin 2189 -> 0 bytes
.../31edf35b-ffca-4f24-8229-f87dc34e3087.dict | Bin 2189 -> 0 bytes
.../652bd393-678a-4f16-a504-fd8ce1229355.dict | Bin 2189 -> 0 bytes
.../792eb972-d046-48e6-9428-f6a3aed92fad.dict | Bin 2189 -> 0 bytes
.../914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict | Bin 2189 -> 0 bytes
.../d25e554e-deac-4e4a-9289-96f0d121d8fc.dict | Bin 2189 -> 0 bytes
.../ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict | Bin 2162 -> 0 bytes
.../hybrid/test_kylin_hybrid_inner_join.json | 13 -
.../hybrid/test_kylin_hybrid_left_join.json | 13 -
.../hybrid/test_kylin_hybrid_ready.json | 17 -
.../localmeta/invertedindex/test_kylin_ii.json | 56 -
.../invertedindex_desc/test_kylin_ii_desc.json | 69 -
.../test_case_data/localmeta/kylin.properties | 77 -
.../test_case_data/localmeta/kylin_job_conf.xml | 74 -
.../model_desc/test_kylin_ii_model_desc.json | 57 -
...est_kylin_with_slr_left_join_model_desc.json | 49 -
.../test_kylin_with_slr_model_desc.json | 42 -
..._kylin_without_slr_left_join_model_desc.json | 42 -
.../test_kylin_without_slr_model_desc.json | 42 -
.../localmeta/project/default.json | 41 -
.../localmeta/project/onlyinner.json | 16 -
.../localmeta/project/onlyleft.json | 16 -
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 151 -
.../table/DEFAULT.TEST_KYLIN_FACT.json | 43 -
.../localmeta/table/EDW.TEST_CAL_DT.json | 407 -
.../table/EDW.TEST_SELLER_TYPE_DIM.json | 43 -
.../localmeta/table/EDW.TEST_SITES.json | 47 -
...6337839-a90f-46fa-a67b-37e330fe9685.snapshot | Bin 588604 -> 0 bytes
...af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot | Bin 588631 -> 0 bytes
...6de9855-8815-4f3b-8ac6-c9386db0b196.snapshot | Bin 588631 -> 0 bytes
...ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot | Bin 588631 -> 0 bytes
...a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot | Bin 588631 -> 0 bytes
...5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot | Bin 588631 -> 0 bytes
...6cbec7f-0326-443d-b45f-54549d5609c6.snapshot | Bin 588631 -> 0 bytes
...91cd3a0-3ae5-4306-87ab-1624d45a6cde.snapshot | Bin 588604 -> 0 bytes
...9a675fc-64d2-40d1-bbd7-492db2510cd1.snapshot | Bin 55219 -> 0 bytes
...9715f33-15c9-4745-83f9-f2b9817d9100.snapshot | Bin 55219 -> 0 bytes
...f8a90f1-bbbe-49ff-937a-2a2a6e5f8615.snapshot | Bin 55219 -> 0 bytes
...172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot | Bin 55219 -> 0 bytes
...fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot | Bin 988 -> 0 bytes
...7238b77-a45d-420f-a249-5f1572ea4d62.snapshot | Bin 988 -> 0 bytes
...43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot | Bin 988 -> 0 bytes
...1b6a60a-14d4-4653-92f7-c96d01191c75.snapshot | Bin 961 -> 0 bytes
...818054b-1a76-4cbd-bdd8-7871a240360c.snapshot | Bin 988 -> 0 bytes
...271e1f2-dd65-4249-a1f2-aca02a651158.snapshot | Bin 21417 -> 0 bytes
...888015f-f059-4ccc-8324-0cdd70b46673.snapshot | Bin 21417 -> 0 bytes
...8130338-fcf4-429e-91b0-cd8dfd397280.snapshot | Bin 21417 -> 0 bytes
...d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot | Bin 21417 -> 0 bytes
...95e0b5b-a399-4f08-8f40-b0fa53b565e0.snapshot | Bin 21398 -> 0 bytes
.../minicluster/hbase-export.tar.gz | Bin 27762040 -> 0 bytes
.../test_case_data/minicluster/kylin.properties | 80 -
.../minicluster/kylin_job_conf.lzo_disabled.xml | 74 -
.../minicluster/kylin_job_conf.lzo_enabled.xml | 74 -
.../minicluster/kylin_job_conf.xml | 74 -
.../sandbox/capacity-scheduler.xml | 114 -
examples/test_case_data/sandbox/core-site.xml | 165 -
.../test_case_data/sandbox/hadoop-policy.xml | 74 -
.../test_case_data/sandbox/hbase-policy.xml | 34 -
examples/test_case_data/sandbox/hbase-site.xml | 194 -
examples/test_case_data/sandbox/hdfs-site.xml | 285 -
examples/test_case_data/sandbox/hive-site.xml | 794 -
.../test_case_data/sandbox/kylin.properties | 144 -
.../test_case_data/sandbox/kylin_job_conf.xml | 74 -
examples/test_case_data/sandbox/mapred-site.xml | 254 -
examples/test_case_data/sandbox/yarn-site.xml | 534 -
.../.settings/org.eclipse.core.resources.prefs | 5 -
.../.settings/org.eclipse.jdt.core.prefs | 379 -
.../.settings/org.eclipse.jdt.ui.prefs | 7 -
invertedindex/pom.xml | 165 -
.../invertedindex/IICapabilityChecker.java | 182 -
.../kylin/invertedindex/IIDescManager.java | 245 -
.../apache/kylin/invertedindex/IIInstance.java | 342 -
.../apache/kylin/invertedindex/IIManager.java | 317 -
.../apache/kylin/invertedindex/IISegment.java | 288 -
.../invertedindex/index/BitMapContainer.java | 278 -
.../index/ColumnValueContainer.java | 43 -
.../index/CompressedValueContainer.java | 179 -
.../invertedindex/index/RawTableRecord.java | 130 -
.../kylin/invertedindex/index/ShardingHash.java | 32 -
.../index/ShardingSliceBuilder.java | 53 -
.../apache/kylin/invertedindex/index/Slice.java | 194 -
.../kylin/invertedindex/index/SliceBuilder.java | 119 -
.../kylin/invertedindex/index/TableRecord.java | 156 -
.../invertedindex/index/TableRecordInfo.java | 185 -
.../index/TableRecordInfoDigest.java | 164 -
.../kylin/invertedindex/model/IIDesc.java | 372 -
.../kylin/invertedindex/model/IIDimension.java | 71 -
.../invertedindex/model/IIKeyValueCodec.java | 253 -
.../apache/kylin/invertedindex/tools/IICLI.java | 106 -
.../invertedindex/IIDescManagerTest.java | 105 -
.../invertedindex/invertedindex/IIDescTest.java | 69 -
.../invertedindex/IIInstanceTest.java | 80 -
.../invertedindex/InvertedIndexLocalTest.java | 232 -
.../invertedindex/invertedindex/LZFTest.java | 50 -
jdbc/.settings/org.eclipse.core.resources.prefs | 5 -
jdbc/.settings/org.eclipse.jdt.core.prefs | 379 -
jdbc/.settings/org.eclipse.jdt.ui.prefs | 7 -
jdbc/pom.xml | 97 -
.../main/java/org/apache/kylin/jdbc/Driver.java | 142 -
.../org/apache/kylin/jdbc/IRemoteClient.java | 56 -
.../java/org/apache/kylin/jdbc/KylinClient.java | 390 -
.../org/apache/kylin/jdbc/KylinConnection.java | 141 -
.../org/apache/kylin/jdbc/KylinJdbcFactory.java | 110 -
.../java/org/apache/kylin/jdbc/KylinMeta.java | 393 -
.../kylin/jdbc/KylinPreparedStatement.java | 123 -
.../org/apache/kylin/jdbc/KylinResultSet.java | 72 -
.../org/apache/kylin/jdbc/KylinStatement.java | 31 -
.../kylin/jdbc/json/PreparedQueryRequest.java | 33 -
.../apache/kylin/jdbc/json/QueryRequest.java | 49 -
.../apache/kylin/jdbc/json/SQLResponseStub.java | 325 -
.../kylin/jdbc/json/StatementParameter.java | 47 -
.../apache/kylin/jdbc/json/TableMetaStub.java | 345 -
.../util/DefaultSslProtocolSocketFactory.java | 148 -
.../jdbc/util/DefaultX509TrustManager.java | 114 -
.../apache/kylin/jdbc/util/Log4jConfigurer.java | 55 -
.../resources/org-apache-kylin-jdbc.properties | 27 -
.../java/org/apache/kylin/jdbc/DriverTest.java | 167 -
.../java/org/apache/kylin/jdbc/DummyClient.java | 83 -
.../java/org/apache/kylin/jdbc/DummyDriver.java | 30 -
.../org/apache/kylin/jdbc/DummyJdbcFactory.java | 34 -
.../apache/kylin/jdbc/SQLResonseStubTest.java | 62 -
job/.gitignore | 1 -
job/.settings/org.eclipse.core.resources.prefs | 6 -
job/.settings/org.eclipse.jdt.core.prefs | 379 -
job/.settings/org.eclipse.jdt.ui.prefs | 7 -
job/dependency-reduced-pom.xml | 594 -
job/pom.xml | 257 -
.../apache/kylin/job/AbstractJobBuilder.java | 97 -
.../apache/kylin/job/CubeMetadataUpgrade.java | 781 -
.../java/org/apache/kylin/job/JobInstance.java | 499 -
.../org/apache/kylin/job/JoinedFlatTable.java | 243 -
.../java/org/apache/kylin/job/Scheduler.java | 37 -
.../apache/kylin/job/cmd/BaseCommandOutput.java | 30 -
.../apache/kylin/job/cmd/ICommandOutput.java | 44 -
.../org/apache/kylin/job/cmd/IJobCommand.java | 32 -
.../java/org/apache/kylin/job/cmd/ShellCmd.java | 117 -
.../apache/kylin/job/cmd/ShellCmdOutput.java | 83 -
.../kylin/job/common/HadoopCmdOutput.java | 109 -
.../kylin/job/common/HadoopShellExecutable.java | 97 -
.../apache/kylin/job/common/HqlExecutable.java | 105 -
.../kylin/job/common/MapReduceExecutable.java | 240 -
.../kylin/job/common/ShellExecutable.java | 144 -
.../kylin/job/constant/BatchConstants.java | 57 -
.../kylin/job/constant/ExecutableConstants.java | 76 -
.../kylin/job/constant/JobStatusEnum.java | 49 -
.../kylin/job/constant/JobStepCmdTypeEnum.java | 27 -
.../kylin/job/constant/JobStepStatusEnum.java | 51 -
.../org/apache/kylin/job/cube/CubingJob.java | 142 -
.../apache/kylin/job/cube/CubingJobBuilder.java | 476 -
.../kylin/job/cube/GarbageCollectionStep.java | 210 -
.../kylin/job/cube/MergeDictionaryStep.java | 198 -
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 149 -
.../job/cube/UpdateCubeInfoAfterMergeStep.java | 151 -
.../org/apache/kylin/job/dao/ExecutableDao.java | 218 -
.../kylin/job/dao/ExecutableOutputPO.java | 67 -
.../org/apache/kylin/job/dao/ExecutablePO.java | 80 -
.../kylin/job/engine/JobEngineConfig.java | 185 -
.../kylin/job/exception/ExecuteException.java | 42 -
.../exception/IllegalStateTranferException.java | 46 -
.../kylin/job/exception/JobException.java | 58 -
.../kylin/job/exception/LockException.java | 45 -
.../job/exception/PersistentException.java | 45 -
.../kylin/job/exception/SchedulerException.java | 45 -
.../kylin/job/execution/AbstractExecutable.java | 307 -
.../kylin/job/execution/ChainedExecutable.java | 32 -
.../job/execution/DefaultChainedExecutable.java | 123 -
.../kylin/job/execution/DefaultOutput.java | 98 -
.../apache/kylin/job/execution/Executable.java | 43 -
.../kylin/job/execution/ExecutableContext.java | 31 -
.../kylin/job/execution/ExecutableState.java | 77 -
.../kylin/job/execution/ExecuteResult.java | 56 -
.../apache/kylin/job/execution/Idempotent.java | 29 -
.../org/apache/kylin/job/execution/Output.java | 35 -
.../kylin/job/hadoop/AbstractHadoopJob.java | 552 -
.../cardinality/ColumnCardinalityMapper.java | 102 -
.../cardinality/ColumnCardinalityReducer.java | 92 -
.../cardinality/HiveColumnCardinalityJob.java | 112 -
.../HiveColumnCardinalityUpdateJob.java | 160 -
.../kylin/job/hadoop/cube/BaseCuboidJob.java | 39 -
.../kylin/job/hadoop/cube/BaseCuboidMapper.java | 244 -
.../kylin/job/hadoop/cube/CubeHFileJob.java | 109 -
.../kylin/job/hadoop/cube/CubeHFileMapper.java | 182 -
.../apache/kylin/job/hadoop/cube/CuboidJob.java | 193 -
.../kylin/job/hadoop/cube/CuboidReducer.java | 116 -
.../cube/FactDistinctColumnsCombiner.java | 57 -
.../job/hadoop/cube/FactDistinctColumnsJob.java | 129 -
.../hadoop/cube/FactDistinctColumnsMapper.java | 138 -
.../hadoop/cube/FactDistinctColumnsReducer.java | 93 -
.../job/hadoop/cube/KeyDistributionJob.java | 157 -
.../job/hadoop/cube/KeyDistributionMapper.java | 124 -
.../job/hadoop/cube/KeyDistributionReducer.java | 113 -
.../kylin/job/hadoop/cube/MergeCuboidJob.java | 103 -
.../job/hadoop/cube/MergeCuboidMapper.java | 247 -
.../job/hadoop/cube/MetadataCleanupJob.java | 186 -
.../kylin/job/hadoop/cube/NDCuboidJob.java | 39 -
.../kylin/job/hadoop/cube/NDCuboidMapper.java | 141 -
.../hadoop/cube/RangeKeyDistributionJob.java | 115 -
.../hadoop/cube/RangeKeyDistributionMapper.java | 71 -
.../cube/RangeKeyDistributionReducer.java | 102 -
.../cube/RowKeyDistributionCheckerJob.java | 97 -
.../cube/RowKeyDistributionCheckerMapper.java | 111 -
.../cube/RowKeyDistributionCheckerReducer.java | 51 -
.../job/hadoop/cube/StorageCleanupJob.java | 312 -
.../job/hadoop/dict/CreateDictionaryJob.java | 66 -
.../dict/CreateInvertedIndexDictionaryJob.java | 63 -
.../kylin/job/hadoop/hbase/BulkLoadJob.java | 100 -
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 202 -
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 175 -
.../job/hadoop/hive/IIJoinedFlatTableDesc.java | 74 -
.../job/hadoop/hive/IJoinedFlatTableDesc.java | 38 -
.../job/hadoop/hive/IntermediateColumnDesc.java | 59 -
.../job/hadoop/hive/SqlHiveDataTypeMapping.java | 46 -
.../job/hadoop/invertedindex/IIBulkLoadJob.java | 83 -
.../hadoop/invertedindex/IICreateHFileJob.java | 91 -
.../invertedindex/IICreateHFileMapper.java | 55 -
.../hadoop/invertedindex/IICreateHTableJob.java | 124 -
.../IIDistinctColumnsCombiner.java | 58 -
.../invertedindex/IIDistinctColumnsJob.java | 136 -
.../invertedindex/IIDistinctColumnsMapper.java | 66 -
.../invertedindex/IIDistinctColumnsReducer.java | 77 -
.../hadoop/invertedindex/IIFlattenHiveJob.java | 96 -
.../hadoop/invertedindex/InvertedIndexJob.java | 140 -
.../invertedindex/InvertedIndexMapper.java | 90 -
.../invertedindex/InvertedIndexPartitioner.java | 73 -
.../invertedindex/InvertedIndexReducer.java | 100 -
.../invertedindex/RandomKeyDistributionJob.java | 115 -
.../RandomKeyDistributionMapper.java | 69 -
.../RandomKeyDistributionReducer.java | 66 -
.../job/impl/threadpool/DefaultContext.java | 63 -
.../job/impl/threadpool/DefaultScheduler.java | 220 -
.../apache/kylin/job/invertedindex/IIJob.java | 51 -
.../kylin/job/invertedindex/IIJobBuilder.java | 214 -
.../kylin/job/manager/ExecutableManager.java | 325 -
.../apache/kylin/job/tools/CleanHtableCLI.java | 76 -
.../kylin/job/tools/CubeMigrationCLI.java | 523 -
.../tools/DefaultSslProtocolSocketFactory.java | 150 -
.../job/tools/DefaultX509TrustManager.java | 114 -
.../kylin/job/tools/DeployCoprocessorCLI.java | 384 -
.../job/tools/GridTableHBaseBenchmark.java | 391 -
.../kylin/job/tools/HadoopStatusChecker.java | 103 -
.../kylin/job/tools/HadoopStatusGetter.java | 255 -
.../kylin/job/tools/HtableAlterMetadataCLI.java | 89 -
.../apache/kylin/job/tools/OptionsHelper.java | 81 -
.../apache/kylin/job/tools/RowCounterCLI.java | 88 -
.../kylin/job/tools/UpdateHTableHostCLI.java | 199 -
.../apache/kylin/job/BaseTestExecutable.java | 32 -
.../kylin/job/BuildCubeWithEngineTest.java | 286 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 255 -
.../java/org/apache/kylin/job/DataGenTest.java | 55 -
.../java/org/apache/kylin/job/DeployUtil.java | 264 -
.../apache/kylin/job/ErrorTestExecutable.java | 42 -
.../org/apache/kylin/job/ExportHBaseData.java | 161 -
.../apache/kylin/job/FailedTestExecutable.java | 42 -
.../apache/kylin/job/SelfStopExecutable.java | 47 -
.../apache/kylin/job/SucceedTestExecutable.java | 42 -
.../apache/kylin/job/dataGen/ColumnConfig.java | 72 -
.../kylin/job/dataGen/FactTableGenerator.java | 646 -
.../org/apache/kylin/job/dataGen/GenConfig.java | 82 -
.../job/hadoop/cube/BaseCuboidJobTest.java | 73 -
.../cube/BaseCuboidMapperPerformanceTest.java | 65 -
.../job/hadoop/cube/BaseCuboidMapperTest.java | 144 -
.../apache/kylin/job/hadoop/cube/CopySeq.java | 81 -
.../job/hadoop/cube/CubeHFileMapper2Test.java | 93 -
.../job/hadoop/cube/CubeHFileMapperTest.java | 80 -
.../kylin/job/hadoop/cube/CubeReducerTest.java | 205 -
.../job/hadoop/cube/KeyDistributionJobTest.java | 81 -
.../hadoop/cube/KeyDistributionMapperTest.java | 171 -
.../job/hadoop/cube/MergeCuboidJobTest.java | 81 -
.../job/hadoop/cube/MergeCuboidMapperTest.java | 187 -
.../kylin/job/hadoop/cube/MockupMapContext.java | 313 -
.../kylin/job/hadoop/cube/NDCuboidJobTest.java | 79 -
.../job/hadoop/cube/NDCuboidMapperTest.java | 115 -
.../cube/RandomKeyDistributionMapperTest.java | 71 -
.../cube/RandomKeyDistributionReducerTest.java | 70 -
.../cube/RangeKeyDistributionJobTest.java | 67 -
.../cube/RangeKeyDistributionMapperTest.java | 112 -
.../cube/RangeKeyDistributionReducerTest.java | 49 -
.../job/hadoop/hbase/CreateHTableTest.java | 67 -
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 102 -
.../job/hadoop/hive/JoinedFlatTableTest.java | 86 -
.../job/impl/threadpool/BaseSchedulerTest.java | 102 -
.../impl/threadpool/DefaultSchedulerTest.java | 151 -
.../job/manager/ExecutableManagerTest.java | 144 -
.../job/tools/ColumnCardinalityJobTest.java | 62 -
.../job/tools/ColumnCardinalityMapperTest.java | 126 -
.../job/tools/ColumnCardinalityReducerTest.java | 113 -
.../kylin/job/tools/CubeMigrationTests.java | 57 -
.../kylin/job/tools/HBaseRowDigestTest.java | 103 -
.../test/resources/data/8d_cuboid/part-r-00000 | Bin 1476517 -> 0 bytes
.../resources/data/base_cuboid/part-r-00000 | Bin 394644 -> 0 bytes
job/src/test/resources/data/flat_table/000000_0 | Bin 110778 -> 0 bytes
.../resources/data/test_cal_dt/part-r-00000 | 366 -
.../expected_result/flat_item/part-r-00000 | Bin 565 -> 0 bytes
.../jarfile/SampleBadJavaProgram.jarfile | Bin 1006 -> 0 bytes
.../resources/jarfile/SampleJavaProgram.jarfile | Bin 1166 -> 0 bytes
.../test/resources/json/dummy_jobinstance.json | 195 -
.../resources/partition_list/_partition.lst | Bin 326 -> 0 bytes
.../test/resources/partition_list/part-r-00000 | Bin 15004 -> 0 bytes
.../.settings/org.eclipse.core.resources.prefs | 6 -
metadata/.settings/org.eclipse.jdt.core.prefs | 379 -
metadata/.settings/org.eclipse.jdt.ui.prefs | 7 -
metadata/pom.xml | 164 -
metadata/src/main/assembly/assemble.xml | 36 -
.../apache/kylin/measure/MeasureAggregator.java | 78 -
.../kylin/measure/MeasureAggregators.java | 91 -
.../org/apache/kylin/measure/MeasureCodec.java | 79 -
.../apache/kylin/measure/MeasureIngester.java | 48 -
.../org/apache/kylin/measure/MeasureType.java | 151 -
.../kylin/measure/MeasureTypeFactory.java | 155 -
.../kylin/measure/basic/BasicMeasureType.java | 152 -
.../kylin/measure/basic/BigDecimalIngester.java | 41 -
.../measure/basic/BigDecimalMaxAggregator.java | 56 -
.../measure/basic/BigDecimalMinAggregator.java | 57 -
.../measure/basic/BigDecimalSumAggregator.java | 53 -
.../kylin/measure/basic/DoubleIngester.java | 46 -
.../measure/basic/DoubleMaxAggregator.java | 55 -
.../measure/basic/DoubleMinAggregator.java | 55 -
.../measure/basic/DoubleSumAggregator.java | 52 -
.../kylin/measure/basic/LongIngester.java | 46 -
.../kylin/measure/basic/LongMaxAggregator.java | 55 -
.../kylin/measure/basic/LongMinAggregator.java | 55 -
.../kylin/measure/basic/LongSumAggregator.java | 52 -
.../kylin/measure/bitmap/BitmapAggregator.java | 57 -
.../kylin/measure/bitmap/BitmapCounter.java | 174 -
.../bitmap/BitmapDistinctCountAggFunc.java | 52 -
.../kylin/measure/bitmap/BitmapMeasureType.java | 109 -
.../kylin/measure/bitmap/BitmapSerializer.java | 81 -
.../kylin/measure/hllc/HLLCAggregator.java | 64 -
.../kylin/measure/hllc/HLLCMeasureType.java | 127 -
.../kylin/measure/hllc/HLLCSerializer.java | 88 -
.../measure/hllc/HLLDistinctCountAggFunc.java | 152 -
.../kylin/metadata/MetadataConstants.java | 54 -
.../apache/kylin/metadata/MetadataManager.java | 422 -
.../metadata/datatype/BigDecimalSerializer.java | 105 -
.../kylin/metadata/datatype/DataType.java | 285 -
.../metadata/datatype/DataTypeSerializer.java | 92 -
.../metadata/datatype/DateTimeSerializer.java | 57 -
.../metadata/datatype/DoubleSerializer.java | 75 -
.../kylin/metadata/datatype/LongSerializer.java | 82 -
.../metadata/datatype/StringSerializer.java | 52 -
.../kylin/metadata/filter/CaseTupleFilter.java | 112 -
.../metadata/filter/ColumnTupleFilter.java | 119 -
.../metadata/filter/CompareTupleFilter.java | 254 -
.../metadata/filter/ConstantTupleFilter.java | 104 -
.../metadata/filter/DynamicTupleFilter.java | 86 -
.../metadata/filter/ExtractTupleFilter.java | 124 -
.../metadata/filter/FunctionTupleFilter.java | 153 -
.../metadata/filter/ITupleFilterTranslator.java | 26 -
.../metadata/filter/LogicalTupleFilter.java | 147 -
.../kylin/metadata/filter/TupleFilter.java | 228 -
.../metadata/filter/TupleFilterSerializer.java | 181 -
.../metadata/filter/util/BuiltInMethod.java | 56 -
.../measure/fixedlen/FixedHLLCodec.java | 75 -
.../measure/fixedlen/FixedLenMeasureCodec.java | 46 -
.../measure/fixedlen/FixedPointLongCodec.java | 80 -
.../apache/kylin/metadata/model/ColumnDesc.java | 136 -
.../kylin/metadata/model/DataModelDesc.java | 248 -
.../kylin/metadata/model/DatabaseDesc.java | 78 -
.../kylin/metadata/model/FunctionDesc.java | 233 -
.../apache/kylin/metadata/model/JoinDesc.java | 137 -
.../apache/kylin/metadata/model/LookupDesc.java | 50 -
.../kylin/metadata/model/MeasureDesc.java | 99 -
.../kylin/metadata/model/ParameterDesc.java | 122 -
.../kylin/metadata/model/PartitionDesc.java | 202 -
.../kylin/metadata/model/SegmentStatusEnum.java | 27 -
.../apache/kylin/metadata/model/TableDesc.java | 171 -
.../apache/kylin/metadata/model/TblColRef.java | 160 -
.../kylin/metadata/project/ProjectInstance.java | 257 -
.../kylin/metadata/project/ProjectL2Cache.java | 275 -
.../kylin/metadata/project/ProjectManager.java | 339 -
.../metadata/project/ProjectStatusEnum.java | 25 -
.../kylin/metadata/project/ProjectTable.java | 95 -
.../metadata/project/RealizationEntry.java | 82 -
.../metadata/realization/CapabilityResult.java | 63 -
.../metadata/realization/IRealization.java | 61 -
.../realization/IRealizationConstants.java | 36 -
.../realization/IRealizationProvider.java | 27 -
.../realization/RealizationRegistry.java | 127 -
.../realization/RealizationStatusEnum.java | 25 -
.../metadata/realization/RealizationType.java | 28 -
.../kylin/metadata/realization/SQLDigest.java | 60 -
.../metadata/realization/SQLDigestUtil.java | 136 -
.../metadata/tool/HiveSourceTableLoader.java | 158 -
.../metadata/tuple/CompoundTupleIterator.java | 63 -
.../org/apache/kylin/metadata/tuple/ITuple.java | 43 -
.../kylin/metadata/tuple/ITupleIterator.java | 53 -
.../org/apache/kylin/metadata/tuple/Tuple.java | 164 -
.../apache/kylin/metadata/tuple/TupleInfo.java | 113 -
.../measure/bitmap/BitmapAggregatorTest.java | 57 -
.../kylin/measure/bitmap/BitmapCounterTest.java | 73 -
.../measure/bitmap/BitmapSerializerTest.java | 55 -
.../kylin/metadata/MetadataManagerTest.java | 68 -
.../datatype/BigDecimalSerializerTest.java | 69 -
.../kylin/metadata/model/TableDescTest.java | 26 -
.../tool/HiveSourceTableLoaderTest.java | 58 -
.../metadata/tool/SandboxMetastoreCLI.java | 67 -
.../metadata/tuple/EmptyTupleIteratorTest.java | 36 -
metadata/src/test/test.json | 181 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
monitor/.settings/org.eclipse.jdt.core.prefs | 379 -
monitor/.settings/org.eclipse.jdt.ui.prefs | 7 -
monitor/pom.xml | 131 -
.../apache/kylin/monitor/ApiRequestParser.java | 241 -
.../java/org/apache/kylin/monitor/Client.java | 59 -
.../org/apache/kylin/monitor/ConfigUtils.java | 246 -
.../org/apache/kylin/monitor/DebugClient.java | 60 -
.../org/apache/kylin/monitor/FileUtils.java | 129 -
.../apache/kylin/monitor/HiveJdbcClient.java | 288 -
.../kylin/monitor/MonitorMetaManager.java | 209 -
.../org/apache/kylin/monitor/QueryParser.java | 272 -
.../org/apache/kylin/monitor/ParseLogTest.java | 73 -
pom.xml | 927 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
query/.settings/org.eclipse.jdt.core.prefs | 379 -
query/.settings/org.eclipse.jdt.ui.prefs | 7 -
query/pom.xml | 294 -
.../java/org/apache/kylin/query/QueryCli.java | 89 -
.../kylin/query/enumerator/CubeEnumerator.java | 241 -
.../kylin/query/enumerator/HiveEnumerator.java | 128 -
.../query/enumerator/LookupTableEnumerator.java | 118 -
.../kylin/query/enumerator/OLAPQuery.java | 64 -
.../kylin/query/optrule/OLAPAggregateRule.java | 70 -
.../kylin/query/optrule/OLAPFilterRule.java | 49 -
.../kylin/query/optrule/OLAPJoinRule.java | 79 -
.../kylin/query/optrule/OLAPLimitRule.java | 58 -
.../kylin/query/optrule/OLAPProjectRule.java | 50 -
.../kylin/query/optrule/OLAPSortRule.java | 50 -
.../optrule/OLAPToEnumerableConverterRule.java | 44 -
.../kylin/query/relnode/ColumnRowType.java | 87 -
.../kylin/query/relnode/OLAPAggregateRel.java | 398 -
.../apache/kylin/query/relnode/OLAPContext.java | 129 -
.../kylin/query/relnode/OLAPFilterRel.java | 327 -
.../apache/kylin/query/relnode/OLAPJoinRel.java | 326 -
.../kylin/query/relnode/OLAPLimitRel.java | 134 -
.../kylin/query/relnode/OLAPProjectRel.java | 303 -
.../org/apache/kylin/query/relnode/OLAPRel.java | 190 -
.../apache/kylin/query/relnode/OLAPSortRel.java | 146 -
.../kylin/query/relnode/OLAPTableScan.java | 275 -
.../relnode/OLAPToEnumerableConverter.java | 121 -
.../apache/kylin/query/routing/Candidate.java | 100 -
.../routing/NoRealizationFoundException.java | 36 -
.../apache/kylin/query/routing/QueryRouter.java | 92 -
.../apache/kylin/query/routing/RoutingRule.java | 111 -
.../routing/rules/RealizationSortRule.java | 34 -
.../rules/RemoveUncapableRealizationsRule.java | 44 -
.../apache/kylin/query/schema/OLAPSchema.java | 128 -
.../kylin/query/schema/OLAPSchemaFactory.java | 130 -
.../apache/kylin/query/schema/OLAPTable.java | 257 -
.../kylin/query/test/CombinationTest.java | 73 -
.../org/apache/kylin/query/test/H2Database.java | 139 -
.../apache/kylin/query/test/IIQueryTest.java | 68 -
.../apache/kylin/query/test/KylinQueryTest.java | 276 -
.../apache/kylin/query/test/KylinTestBase.java | 474 -
.../query/test/RealizationRegistryTest.java | 58 -
query/src/test/resources/logging.properties | 22 -
query/src/test/resources/query/h2/query07.sql | 21 -
query/src/test/resources/query/h2/query09.sql | 23 -
query/src/test/resources/query/h2/query10.sql | 25 -
query/src/test/resources/query/sql/query00.sql | 22 -
query/src/test/resources/query/sql/query01.sql | 20 -
query/src/test/resources/query/sql/query02.sql | 19 -
query/src/test/resources/query/sql/query03.sql | 20 -
query/src/test/resources/query/sql/query04.sql | 21 -
query/src/test/resources/query/sql/query05.sql | 22 -
query/src/test/resources/query/sql/query06.sql | 23 -
query/src/test/resources/query/sql/query07.sql | 22 -
query/src/test/resources/query/sql/query08.sql | 22 -
query/src/test/resources/query/sql/query09.sql | 23 -
query/src/test/resources/query/sql/query10.sql | 26 -
query/src/test/resources/query/sql/query11.sql | 26 -
query/src/test/resources/query/sql/query12.sql | 25 -
query/src/test/resources/query/sql/query13.sql | 24 -
query/src/test/resources/query/sql/query14.sql | 35 -
query/src/test/resources/query/sql/query15.sql | 30 -
query/src/test/resources/query/sql/query16.sql | 34 -
query/src/test/resources/query/sql/query17.sql | 36 -
query/src/test/resources/query/sql/query18.sql | 32 -
query/src/test/resources/query/sql/query19.sql | 36 -
query/src/test/resources/query/sql/query20.sql | 32 -
query/src/test/resources/query/sql/query21.sql | 40 -
query/src/test/resources/query/sql/query22.sql | 42 -
query/src/test/resources/query/sql/query23.sql | 41 -
query/src/test/resources/query/sql/query24.sql | 45 -
query/src/test/resources/query/sql/query25.sql | 46 -
query/src/test/resources/query/sql/query26.sql | 48 -
query/src/test/resources/query/sql/query27.sql | 47 -
query/src/test/resources/query/sql/query28.sql | 30 -
query/src/test/resources/query/sql/query29.sql | 32 -
query/src/test/resources/query/sql/query30.sql | 19 -
query/src/test/resources/query/sql/query31.sql | 40 -
query/src/test/resources/query/sql/query32.sql | 39 -
query/src/test/resources/query/sql/query33.sql | 43 -
query/src/test/resources/query/sql/query34.sql | 41 -
query/src/test/resources/query/sql/query35.sql | 41 -
query/src/test/resources/query/sql/query36.sql | 34 -
query/src/test/resources/query/sql/query37.sql | 36 -
query/src/test/resources/query/sql/query38.sql | 22 -
query/src/test/resources/query/sql/query39.sql | 20 -
query/src/test/resources/query/sql/query40.sql | 22 -
query/src/test/resources/query/sql/query41.sql | 30 -
query/src/test/resources/query/sql/query42.sql | 23 -
query/src/test/resources/query/sql/query43.sql | 25 -
query/src/test/resources/query/sql/query44.sql | 28 -
query/src/test/resources/query/sql/query45.sql | 24 -
query/src/test/resources/query/sql/query46.sql | 19 -
query/src/test/resources/query/sql/query47.sql | 19 -
query/src/test/resources/query/sql/query48.sql | 19 -
query/src/test/resources/query/sql/query49.sql | 36 -
query/src/test/resources/query/sql/query50.sql | 25 -
query/src/test/resources/query/sql/query51.sql | 25 -
query/src/test/resources/query/sql/query52.sql | 22 -
query/src/test/resources/query/sql/query53.sql | 24 -
query/src/test/resources/query/sql/query54.sql | 25 -
query/src/test/resources/query/sql/query55.sql | 19 -
query/src/test/resources/query/sql/query56.sql | 26 -
query/src/test/resources/query/sql/query57.sql | 26 -
query/src/test/resources/query/sql/query58.sql | 26 -
query/src/test/resources/query/sql/query59.sql | 19 -
query/src/test/resources/query/sql/query60.sql | 24 -
query/src/test/resources/query/sql/query61.sql | 24 -
query/src/test/resources/query/sql/query62.sql | 21 -
query/src/test/resources/query/sql/query63.sql | 20 -
query/src/test/resources/query/sql/query64.sql | 21 -
query/src/test/resources/query/sql/query65.sql | 19 -
query/src/test/resources/query/sql/query66.sql | 20 -
query/src/test/resources/query/sql/query67.sql | 22 -
query/src/test/resources/query/sql/query68.sql | 26 -
query/src/test/resources/query/sql/query69.sql | 25 -
query/src/test/resources/query/sql/query70.sql | 24 -
query/src/test/resources/query/sql/query71.sql | 24 -
query/src/test/resources/query/sql/query72.sql | 24 -
query/src/test/resources/query/sql/query73.sql | 19 -
query/src/test/resources/query/sql/query74.sql | 19 -
query/src/test/resources/query/sql/query75.sql | 22 -
query/src/test/resources/query/sql/query76.sql | 22 -
query/src/test/resources/query/sql/query77.sql | 21 -
query/src/test/resources/query/sql/query78.sql | 22 -
query/src/test/resources/query/sql/query79.sql | 19 -
query/src/test/resources/query/sql/query80.sql | 27 -
query/src/test/resources/query/sql/query81.sql | 28 -
query/src/test/resources/query/sql/query82.sql | 22 -
query/src/test/resources/query/sql/query83.sql | 22 -
query/src/test/resources/query/sql/query84.sql | 22 -
query/src/test/resources/query/sql/query85.sql | 22 -
query/src/test/resources/query/sql/query86.sql | 30 -
query/src/test/resources/query/sql/query87.sql | 30 -
query/src/test/resources/query/sql/query88.sql | 30 -
query/src/test/resources/query/sql/query89.sql | 30 -
.../resources/query/sql_all_types/query00.sql | 19 -
.../resources/query/sql_all_types/query01.sql | 19 -
.../resources/query/sql_all_types/query02.sql | 19 -
.../resources/query/sql_all_types/query03.sql | 19 -
.../resources/query/sql_all_types/query04.sql | 19 -
.../resources/query/sql_all_types/query05.sql | 19 -
.../resources/query/sql_all_types/query06.sql | 19 -
.../resources/query/sql_all_types/query07.sql | 19 -
.../resources/query/sql_all_types/query08.sql | 19 -
.../resources/query/sql_all_types/query09.sql | 19 -
.../resources/query/sql_casewhen/query01.sql | 26 -
.../resources/query/sql_casewhen/query02.sql | 22 -
.../resources/query/sql_casewhen/query03.sql | 25 -
.../query/sql_casewhen/query55.sql.disable | 22 -
.../resources/query/sql_casewhen/query56.sql | 22 -
.../resources/query/sql_derived/query01.sql | 47 -
.../resources/query/sql_derived/query02.sql | 47 -
.../resources/query/sql_derived/query03.sql | 21 -
.../resources/query/sql_derived/query04.sql | 20 -
.../resources/query/sql_derived/query05.sql | 23 -
.../resources/query/sql_derived/query06.sql | 20 -
.../resources/query/sql_derived/query07.sql | 22 -
.../query/sql_derived/query08.sql.disabled | 19 -
.../resources/query/sql_derived/query09.sql | 19 -
.../resources/query/sql_derived/query10.sql | 44 -
.../resources/query/sql_distinct/query00.sql | 24 -
.../resources/query/sql_distinct/query01.sql | 25 -
.../resources/query/sql_distinct/query02.sql | 26 -
.../resources/query/sql_distinct/query03.sql | 26 -
.../resources/query/sql_distinct/query04.sql | 27 -
.../resources/query/sql_distinct/query05.sql | 25 -
.../resources/query/sql_distinct/query06.sql | 26 -
.../resources/query/sql_distinct/query07.sql | 24 -
.../query/sql_distinct_precisely/query00.sql | 24 -
.../query/sql_distinct_precisely/query01.sql | 25 -
.../query/sql_distinct_precisely/query02.sql | 26 -
.../query/sql_distinct_precisely/query03.sql | 26 -
.../query/sql_distinct_precisely/query04.sql | 27 -
.../query/sql_distinct_precisely/query05.sql | 25 -
.../query/sql_distinct_precisely/query06.sql | 26 -
.../query/sql_distinct_precisely/query07.sql | 24 -
.../resources/query/sql_dynamic/query01.dat | 2 -
.../resources/query/sql_dynamic/query01.sql | 28 -
.../resources/query/sql_dynamic/query02.dat | 2 -
.../resources/query/sql_dynamic/query02.sql | 30 -
.../resources/query/sql_dynamic/query03.dat | 2 -
.../resources/query/sql_dynamic/query03.sql | 26 -
.../resources/query/sql_fast_common/query00.sql | 23 -
.../test/resources/query/sql_hive/query01.sql | 20 -
.../test/resources/query/sql_hive/query10.sql | 26 -
.../src/test/resources/query/sql_ii/query01.sql | 19 -
.../src/test/resources/query/sql_ii/query02.sql | 19 -
.../src/test/resources/query/sql_ii/query03.sql | 19 -
.../src/test/resources/query/sql_ii/query04.sql | 19 -
.../src/test/resources/query/sql_ii/query05.sql | 19 -
.../resources/query/sql_ii/query06.sql.disabled | 19 -
.../resources/query/sql_ii/query07.sql.disabled | 19 -
.../src/test/resources/query/sql_ii/query08.sql | 19 -
.../src/test/resources/query/sql_ii/query09.sql | 19 -
.../query/sql_invalid/query29_invalid_SQL.sql | 55 -
.../query/sql_invalid/query30_invalid_SQL.sql | 54 -
.../query/sql_invalid/query31_invalid_SQL.sql | 55 -
.../query/sql_invalid/query32_invalid_SQL.sql | 56 -
.../query/sql_invalid/query33_invalid_SQL.sql | 56 -
.../query/sql_invalid/query34_invalid_SQL.sql | 24 -
...ery_count_distinct_on_dimension.sql.disabled | 20 -
.../test/resources/query/sql_lookup/query01.sql | 24 -
.../test/resources/query/sql_lookup/query02.sql | 19 -
.../test/resources/query/sql_lookup/query03.sql | 19 -
.../test/resources/query/sql_lookup/query04.sql | 19 -
.../query/sql_optimize/enable-limit01.sql | 19 -
.../resources/query/sql_orderby/query01.sql | 28 -
.../resources/query/sql_orderby/query02.sql | 25 -
.../resources/query/sql_orderby/query03.sql | 24 -
.../resources/query/sql_subquery/query00.sql | 22 -
.../resources/query/sql_subquery/query01.sql | 26 -
.../query/sql_subquery/query02.sql.disable | 25 -
.../resources/query/sql_subquery/query03.sql | 27 -
.../resources/query/sql_subquery/query04.sql | 27 -
.../resources/query/sql_subquery/query05.sql | 28 -
.../resources/query/sql_subquery/query06.sql | 24 -
.../resources/query/sql_subquery/query07.sql | 24 -
.../resources/query/sql_subquery/query08.sql | 32 -
.../resources/query/sql_tableau/query00.sql | 23 -
.../resources/query/sql_tableau/query01.sql | 33 -
.../resources/query/sql_tableau/query02.sql | 30 -
.../resources/query/sql_tableau/query03.sql | 35 -
.../resources/query/sql_tableau/query04.sql | 39 -
.../resources/query/sql_tableau/query05.sql | 40 -
.../resources/query/sql_tableau/query06.sql | 40 -
.../resources/query/sql_tableau/query07.sql | 26 -
.../query/sql_tableau/query08.sql.disabled | 24 -
.../query/sql_tableau/query09.sql.disabled | 57 -
.../resources/query/sql_tableau/query10.sql | 23 -
.../resources/query/sql_tableau/query11.sql | 24 -
.../resources/query/sql_tableau/query12.sql | 23 -
.../resources/query/sql_tableau/query13.sql | 22 -
.../resources/query/sql_tableau/query14.sql | 19 -
.../resources/query/sql_tableau/query15.sql | 22 -
.../resources/query/sql_tableau/query16.sql | 22 -
.../resources/query/sql_tableau/query17.sql | 19 -
.../resources/query/sql_tableau/query18.sql | 21 -
.../resources/query/sql_tableau/query19.sql | 26 -
.../resources/query/sql_tableau/query20.sql | 26 -
.../resources/query/sql_tableau/query21.sql | 27 -
.../query/sql_tableau/query22.sql.disabled | 28 -
.../resources/query/sql_tableau/query23.sql | 23 -
.../resources/query/sql_tableau/query24.sql | 24 -
.../resources/query/sql_tableau/query25.sql | 28 -
.../resources/query/sql_tableau/query27.sql | 24 -
.../resources/query/sql_tableau/query28.sql | 23 -
.../resources/query/sql_tableau/query67.sql | 23 -
.../resources/query/sql_verifyCount/query04.sql | 19 -
.../query/sql_verifyCount/query04.sql.expected | 1 -
.../resources/query/sql_verifyCount/query05.sql | 19 -
.../query/sql_verifyCount/query05.sql.expected | 1 -
.../resources/query/sql_verifyCount/query06.sql | 19 -
.../query/sql_verifyCount/query06.sql.expected | 1 -
.../resources/query/sql_verifyCount/query07.sql | 19 -
.../query/sql_verifyCount/query07.sql.expected | 1 -
.../resources/query/sql_verifyCount/query08.sql | 19 -
.../query/sql_verifyCount/query08.sql.expected | 1 -
.../resources/query/tableau_probing/query01.sql | 19 -
.../resources/query/tableau_probing/query02.sql | 19 -
.../resources/query/tableau_probing/query03.sql | 19 -
.../resources/query/tableau_probing/query04.sql | 19 -
.../resources/query/tableau_probing/query05.sql | 19 -
.../resources/query/tableau_probing/query06.sql | 19 -
.../resources/query/tableau_probing/query07.sql | 19 -
query/src/test/resources/query/temp/.gitignore | 1 -
script/build.sh | 33 -
script/compress.sh | 47 -
script/download-tomcat.sh | 45 -
script/package.sh | 81 -
script/prepare.sh | 61 -
.../.settings/org.eclipse.core.resources.prefs | 5 -
server/.settings/org.eclipse.jdt.core.prefs | 379 -
server/.settings/org.eclipse.jdt.ui.prefs | 7 -
server/ServiceConfig.md | 38 -
server/pom.xml | 487 -
.../java/org/apache/kylin/rest/DebugTomcat.java | 136 -
.../apache/kylin/rest/constant/Constant.java | 50 -
.../kylin/rest/controller/AccessController.java | 118 -
.../kylin/rest/controller/AdminController.java | 97 -
.../kylin/rest/controller/BasicController.java | 71 -
.../kylin/rest/controller/CacheController.java | 84 -
.../kylin/rest/controller/CubeController.java | 654 -
.../rest/controller/CubeDescController.java | 71 -
.../kylin/rest/controller/JobController.java | 205 -
.../kylin/rest/controller/ModelController.java | 81 -
.../rest/controller/PerformanceController.java | 113 -
.../rest/controller/ProjectController.java | 115 -
.../kylin/rest/controller/QueryController.java | 243 -
.../kylin/rest/controller/TableController.java | 241 -
.../kylin/rest/controller/UserController.java | 67 -
.../rest/exception/BadRequestException.java | 61 -
.../rest/exception/ForbiddenException.java | 41 -
.../rest/exception/InternalErrorException.java | 63 -
.../kylin/rest/exception/NotFoundException.java | 35 -
.../kylin/rest/filter/KylinApiFilter.java | 140 -
.../org/apache/kylin/rest/model/ColumnMeta.java | 265 -
.../java/org/apache/kylin/rest/model/Query.java | 121 -
.../kylin/rest/model/SelectedColumnMeta.java | 143 -
.../org/apache/kylin/rest/model/TableMeta.java | 151 -
.../kylin/rest/request/AccessRequest.java | 67 -
.../kylin/rest/request/CardinalityRequest.java | 77 -
.../rest/request/CreateProjectRequest.java | 47 -
.../apache/kylin/rest/request/CubeRequest.java | 133 -
.../kylin/rest/request/JobBuildRequest.java | 67 -
.../kylin/rest/request/JobListRequest.java | 78 -
.../apache/kylin/rest/request/MetaRequest.java | 42 -
.../kylin/rest/request/MetricsRequest.java | 59 -
.../kylin/rest/request/PrepareSqlRequest.java | 121 -
.../apache/kylin/rest/request/SQLRequest.java | 130 -
.../kylin/rest/request/SaveSqlRequest.java | 72 -
.../kylin/rest/request/UpdateConfigRequest.java | 46 -
.../rest/request/UpdateProjectRequest.java | 56 -
.../rest/response/AccessEntryResponse.java | 70 -
.../kylin/rest/response/ErrorResponse.java | 38 -
.../kylin/rest/response/GeneralResponse.java | 31 -
.../kylin/rest/response/HBaseResponse.java | 108 -
.../kylin/rest/response/MetricsResponse.java | 55 -
.../apache/kylin/rest/response/SQLResponse.java | 144 -
.../kylin/rest/response/TableDescResponse.java | 80 -
.../kylin/rest/security/AclEntityFactory.java | 56 -
.../kylin/rest/security/AclPermission.java | 42 -
.../rest/security/AclPermissionFactory.java | 77 -
.../rest/security/AuthoritiesPopulator.java | 75 -
.../kylin/rest/security/CrossDomainFilter.java | 74 -
.../kylin/rest/security/LdapProvider.java | 104 -
.../security/PasswordPlaceholderConfigurer.java | 89 -
.../rest/security/UnauthorisedEntryPoint.java | 44 -
.../apache/kylin/rest/security/UserManager.java | 29 -
.../kylin/rest/service/AccessService.java | 293 -
.../apache/kylin/rest/service/AclService.java | 467 -
.../apache/kylin/rest/service/AdminService.java | 108 -
.../kylin/rest/service/BadQueryDetector.java | 183 -
.../apache/kylin/rest/service/BasicService.java | 243 -
.../apache/kylin/rest/service/CacheService.java | 159 -
.../apache/kylin/rest/service/CubeService.java | 604 -
.../apache/kylin/rest/service/JobService.java | 324 -
.../kylin/rest/service/PerformService.java | 124 -
.../kylin/rest/service/ProjectService.java | 119 -
.../apache/kylin/rest/service/QueryService.java | 493 -
.../apache/kylin/rest/service/UserService.java | 242 -
.../apache/kylin/rest/util/ClasspathUtil.java | 44 -
.../org/apache/kylin/rest/util/HiveReroute.java | 142 -
.../kylin/rest/util/Log4jConfigListener.java | 46 -
.../org/apache/kylin/rest/util/QueryUtil.java | 219 -
.../org/apache/kylin/rest/util/Serializer.java | 51 -
server/src/main/resources/.gitignore | 2 -
.../src/main/resources/applicationContext.xml | 116 -
server/src/main/resources/dbSqls/schema.sql | 163 -
server/src/main/resources/ehcache-test.xml | 37 -
server/src/main/resources/ehcache.xml | 37 -
.../resources/kylin-server-log4j.properties | 51 -
server/src/main/resources/kylinSecurity.xml | 162 -
.../src/main/webapp/WEB-INF/kylin-servlet.xml | 28 -
server/src/main/webapp/WEB-INF/web.xml | 117 -
server/src/main/webapp/index.html | 27 -
.../org/apache/kylin/jdbc/JDBCDriverTest.java | 270 -
.../org/apache/kylin/rest/bean/BeanTest.java | 74 -
.../apache/kylin/rest/bean/BeanValidator.java | 160 -
.../broadcaster/BroadcasterReceiveServlet.java | 78 -
.../rest/controller/AccessControllerTest.java | 87 -
.../rest/controller/AdminControllerTest.java | 56 -
.../rest/controller/BaseControllerTest.java | 69 -
.../rest/controller/CacheControllerTest.java | 52 -
.../rest/controller/CubeControllerTest.java | 228 -
.../rest/controller/JobControllerTest.java | 125 -
.../rest/controller/ProjectControllerTest.java | 107 -
.../rest/controller/QueryControllerTest.java | 72 -
.../rest/controller/TableControllerTest.java | 73 -
.../rest/controller/UserControllerTest.java | 65 -
.../kylin/rest/service/AccessServiceTest.java | 139 -
.../rest/service/BadQueryDetectorTest.java | 69 -
.../kylin/rest/service/CacheServiceTest.java | 460 -
.../kylin/rest/service/CubeServiceTest.java | 56 -
.../kylin/rest/service/JobServiceTest.java | 45 -
.../kylin/rest/service/QueryServiceTest.java | 61 -
.../kylin/rest/service/ServiceTestBase.java | 88 -
.../kylin/rest/service/UserServiceTest.java | 57 -
src/main/config/assemblies/source-assembly.xml | 119 -
.../checkstyle/checkstyle-java-header.txt | 17 -
src/main/config/checkstyle/checkstyle.xml | 44 -
src/main/config/checkstyle/suppressions.xml | 32 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
storage/.settings/org.eclipse.jdt.core.prefs | 379 -
storage/.settings/org.eclipse.jdt.ui.prefs | 7 -
storage/pom.xml | 236 -
.../apache/kylin/storage/IStorageEngine.java | 33 -
.../apache/kylin/storage/StorageContext.java | 202 -
.../kylin/storage/StorageEngineFactory.java | 45 -
.../storage/filter/BitMapFilterEvaluator.java | 244 -
.../kylin/storage/hbase/ColumnValueRange.java | 212 -
.../storage/hbase/CubeSegmentTupleIterator.java | 490 -
.../kylin/storage/hbase/CubeStorageEngine.java | 677 -
.../kylin/storage/hbase/CubeTupleConverter.java | 139 -
.../storage/hbase/DerivedFilterTranslator.java | 214 -
.../storage/hbase/FuzzyValueCombination.java | 135 -
.../storage/hbase/HBaseClientKVIterator.java | 93 -
.../kylin/storage/hbase/HBaseKeyRange.java | 289 -
.../hbase/InvertedIndexStorageEngine.java | 57 -
.../kylin/storage/hbase/PingHBaseCLI.java | 88 -
.../storage/hbase/RegionScannerAdapter.java | 97 -
.../storage/hbase/ResultScannerAdapter.java | 100 -
.../storage/hbase/ScanOutOfLimitException.java | 31 -
.../hbase/SerializedHBaseTupleIterator.java | 133 -
.../storage/hbase/coprocessor/AggrKey.java | 133 -
.../hbase/coprocessor/AggregationCache.java | 74 -
.../hbase/coprocessor/CoprocessorConstants.java | 24 -
.../hbase/coprocessor/CoprocessorFilter.java | 227 -
.../hbase/coprocessor/CoprocessorProjector.java | 127 -
.../hbase/coprocessor/CoprocessorRowType.java | 155 -
.../CoprocessorTupleFilterTranslator.java | 157 -
.../endpoint/EndpointAggregationCache.java | 47 -
.../endpoint/EndpointAggregators.java | 282 -
.../coprocessor/endpoint/EndpointEnabler.java | 53 -
.../endpoint/EndpointTupleIterator.java | 354 -
.../endpoint/HbaseServerKVIterator.java | 96 -
.../hbase/coprocessor/endpoint/IIEndpoint.java | 194 -
.../coprocessor/endpoint/IIResponseAdapter.java | 25 -
.../endpoint/SliceBitMapProvider.java | 57 -
.../endpoint/generated/IIProtos.java | 2205 -
.../coprocessor/endpoint/protobuf/II.proto | 46 -
.../observer/AggregateRegionObserver.java | 108 -
.../observer/AggregationScanner.java | 170 -
.../observer/ObserverAggregationCache.java | 161 -
.../observer/ObserverAggregators.java | 273 -
.../coprocessor/observer/ObserverBehavior.java | 26 -
.../coprocessor/observer/ObserverEnabler.java | 189 -
.../coprocessor/observer/ObserverTuple.java | 97 -
.../kylin/storage/hybrid/HybridInstance.java | 273 -
.../kylin/storage/hybrid/HybridManager.java | 159 -
.../storage/hybrid/HybridStorageEngine.java | 60 -
.../apache/kylin/storage/MiniClusterTest.java | 63 -
.../filter/BitMapFilterEvaluatorTest.java | 240 -
.../kylin/storage/filter/FilterBaseTest.java | 215 -
.../storage/filter/FilterEvaluateTest.java | 182 -
.../kylin/storage/filter/FilterPerfTest.java | 154 -
.../storage/filter/FilterSerializeTest.java | 204 -
.../storage/hbase/ColumnValueRangeTest.java | 130 -
.../storage/hbase/CubeStorageEngineTest.java | 142 -
.../hbase/FuzzyValueCombinationTest.java | 118 -
.../storage/hbase/InvertedIndexHBaseTest.java | 115 -
.../hbase/coprocessor/RowProjectorTest.java | 88 -
.../endpoint/EndpointAggregationTest.java | 205 -
.../endpoint/TableRecordInfoTest.java | 59 -
.../observer/AggregateRegionObserverTest.java | 345 -
.../observer/RowAggregatorsTest.java | 62 -
.../hbase/coprocessor/observer/RowTypeTest.java | 66 -
.../kylin/storage/hybrid/HybridManagerTest.java | 60 -
.../storage/minicluster/HiveJDBCClientTest.java | 183 -
.../minicluster/HiveMiniClusterTest.java | 131 -
.../apache/kylin/storage/test/StorageTest.java | 255 -
webapp/.bowerrc | 3 -
webapp/.editorconfig | 38 -
webapp/.gitignore | 8 -
webapp/.jshintrc | 22 -
webapp/Gruntfile.js | 290 -
webapp/README.md | 4 -
webapp/app/css/AdminLTE.css | 4112 -
webapp/app/htaccess.dist | 20 -
webapp/app/image/ajax-loader.gif | Bin 847 -> 0 bytes
webapp/app/image/database.png | Bin 3147 -> 0 bytes
webapp/app/image/default_avatar.png | Bin 977 -> 0 bytes
webapp/app/image/favicon.ico | Bin 16958 -> 0 bytes
webapp/app/image/icon_table.png | Bin 484 -> 0 bytes
webapp/app/image/logo.png | Bin 114812 -> 0 bytes
webapp/app/image/table-icon.png | Bin 489 -> 0 bytes
webapp/app/image/waiting.gif | Bin 1787 -> 0 bytes
webapp/app/index.html | 193 -
webapp/app/js/app.js | 20 -
webapp/app/js/config.js | 157 -
webapp/app/js/controllers/access.js | 129 -
webapp/app/js/controllers/admin.js | 252 -
webapp/app/js/controllers/auth.js | 47 -
webapp/app/js/controllers/cube.js | 104 -
webapp/app/js/controllers/cubeAdvanceSetting.js | 99 -
webapp/app/js/controllers/cubeDimensions.js | 401 -
webapp/app/js/controllers/cubeEdit.js | 655 -
webapp/app/js/controllers/cubeFilter.js | 23 -
webapp/app/js/controllers/cubeModel.js | 157 -
webapp/app/js/controllers/cubeRefresh.js | 23 -
webapp/app/js/controllers/cubeSchema.js | 282 -
webapp/app/js/controllers/cubes.js | 540 -
webapp/app/js/controllers/dashboard.js | 237 -
webapp/app/js/controllers/job.js | 220 -
webapp/app/js/controllers/page.js | 284 -
webapp/app/js/controllers/projectMeta.js | 120 -
webapp/app/js/controllers/projects.js | 96 -
webapp/app/js/controllers/query.js | 514 -
webapp/app/js/controllers/sourceMeta.js | 339 -
.../app/js/directives/angular-tree-control.js | 363 -
webapp/app/js/directives/directives.js | 231 -
webapp/app/js/factories/graph.js | 77 -
webapp/app/js/filters/filter.js | 176 -
webapp/app/js/listeners.js | 98 -
webapp/app/js/model/cubeConfig.js | 78 -
webapp/app/js/model/cubeDescModel.js | 75 -
webapp/app/js/model/cubeListModel.js | 73 -
webapp/app/js/model/cubesManager.js | 22 -
webapp/app/js/model/jobConfig.js | 36 -
webapp/app/js/model/jobListModel.js | 51 -
webapp/app/js/model/metaModel.js | 80 -
webapp/app/js/model/projectConfig.js | 27 -
webapp/app/js/model/projectModel.js | 74 -
webapp/app/js/model/tableConfig.js | 27 -
webapp/app/js/model/tableModel.js | 127 -
webapp/app/js/services/access.js | 26 -
webapp/app/js/services/admin.js | 26 -
webapp/app/js/services/auth.js | 25 -
webapp/app/js/services/cache.js | 23 -
webapp/app/js/services/cubeDesc.js | 23 -
webapp/app/js/services/cubes.js | 37 -
webapp/app/js/services/dashboard.js | 32 -
webapp/app/js/services/graph.js | 46 -
webapp/app/js/services/jobs.js | 27 -
webapp/app/js/services/kylinProperties.js | 93 -
webapp/app/js/services/message.js | 57 -
webapp/app/js/services/model.js | 23 -
webapp/app/js/services/ngLoading.js | 52 -
webapp/app/js/services/projects.js | 26 -
webapp/app/js/services/query.js | 29 -
webapp/app/js/services/tables.js | 29 -
webapp/app/js/services/tree.js | 325 -
webapp/app/js/services/users.js | 66 -
webapp/app/js/utils/utils.js | 68 -
webapp/app/less/animation.less | 46 -
webapp/app/less/app.less | 631 -
webapp/app/less/build.less | 23 -
webapp/app/less/component.less | 915 -
webapp/app/less/home.less | 102 -
webapp/app/less/layout.less | 49 -
webapp/app/less/navbar.less | 225 -
webapp/app/partials/admin/admin.html | 177 -
webapp/app/partials/common/access.html | 110 -
.../cubeDesigner/advanced_settings.html | 255 -
.../app/partials/cubeDesigner/data_model.html | 199 -
.../app/partials/cubeDesigner/dimensions.html | 354 -
webapp/app/partials/cubeDesigner/filter.html | 66 -
.../app/partials/cubeDesigner/incremental.html | 112 -
webapp/app/partials/cubeDesigner/info.html | 112 -
webapp/app/partials/cubeDesigner/measures.html | 206 -
webapp/app/partials/cubeDesigner/overview.html | 76 -
webapp/app/partials/cubes/cube_clone.html | 63 -
webapp/app/partials/cubes/cube_detail.html | 124 -
webapp/app/partials/cubes/cube_edit.html | 34 -
webapp/app/partials/cubes/cube_json_edit.html | 61 -
webapp/app/partials/cubes/cube_schema.html | 68 -
webapp/app/partials/cubes/cubes.html | 166 -
webapp/app/partials/dashboard.html | 228 -
webapp/app/partials/directives/loading.html | 24 -
webapp/app/partials/directives/noResult.html | 21 -
webapp/app/partials/directives/pagination.html | 27 -
webapp/app/partials/directives/typeahead.html | 30 -
webapp/app/partials/footer.html | 28 -
webapp/app/partials/header.html | 77 -
webapp/app/partials/jobs/job_merge.html | 132 -
webapp/app/partials/jobs/job_refresh.html | 99 -
webapp/app/partials/jobs/job_steps.html | 159 -
webapp/app/partials/jobs/job_submit.html | 78 -
webapp/app/partials/jobs/jobs.html | 173 -
webapp/app/partials/login.html | 73 -
.../app/partials/projects/project_create.html | 56 -
.../app/partials/projects/project_detail.html | 64 -
.../partials/projects/project_table_tree.html | 38 -
webapp/app/partials/projects/projects.html | 84 -
webapp/app/partials/query/query.html | 238 -
webapp/app/partials/query/query_detail.html | 272 -
webapp/app/partials/tables/source_metadata.html | 170 -
.../app/partials/tables/source_table_tree.html | 89 -
webapp/app/routes.json | 99 -
webapp/bower.json | 49 -
webapp/grunt.json | 71 -
webapp/package.json | 22 -
1314 files changed, 745494 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 147728d..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,105 +0,0 @@
-### Eclipse
-*.pydevproject
-.metadata
-.gradle
-tmp/
-*.tmp
-*.bak
-*.swp
-*~.nib
-
-local.properties
-release.properties
-.loadpath
-
-# maven release files
-*.versionsBackup
-**/pom.xml.tag
-
-#IDEA
-*.iml
-#.settings
-
-# External tool builders
-.externalToolBuilders/
-
-# Locally stored "Eclipse launch configurations"
-*.launch
-
-# CDT-specific
-.cproject
-
-# PDT-specific
-.buildpath
-
-# sbteclipse plugin
-.target
-
-# TeXlipse plugin
-.texlipse
-
-### Java
-*.class
-
-# Mobile Tools for Java (J2ME)
-.mtj.tmp/
-
-# Package Files #
-*.jar
-*.war
-*.ear
-*.tgz
-*.tar.gz
-
-
-
-# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
-hs_err_pid*
-
-
-### Origin
-# /
-.classpath
-.project
-.idea
-**/*.DS_Store
-*.releaseBackup
-*.log
-jdbc/kylin_jdbc.log.*
-*.DS_Store
-
-
-
-# .settings
-org.eclipse.m2e.core.prefs
-org.eclipse.wst.common.component
-org.eclipse.wst.common.project.facet.core.xml
-org.eclipse.wst.validation.prefs
-server/.settings/.jsdtscope
-server/.settings/org.eclipse.wst.common.project.facet.core.prefs.xml
-server/.settings/org.eclipse.wst.jsdt.ui.superType.container
-server/.settings/org.eclipse.wst.jsdt.ui.superType.name
-
-#web
-webapp/app/WEB-INF/*
-webapp/dist
-webapp/node_modules:
-webapp/CHANGELOG.md
-
-# generated files
-target
-logs
-
-# package file and folder
-dist
-tomcat
-lib
-webapp/app/components/*
-commit.sha1
-
-# website
-website/_site
-website/.sass-cache
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/.reviewboardrc
----------------------------------------------------------------------
diff --git a/.reviewboardrc b/.reviewboardrc
deleted file mode 100644
index 79d75f5..0000000
--- a/.reviewboardrc
+++ /dev/null
@@ -1,4 +0,0 @@
-REVIEWBOARD_URL = "https://reviews.apache.org"
-REPOSITORY = "kylin"
-BRANCH = "1.x-staging"
-LAND_DEST_BRANCH = "1.x-staging"
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 99f26c0..0000000
--- a/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,2 +0,0 @@
-eclipse.preferences.version=1
-encoding/<project>=UTF-8
[24/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv b/examples/test_case_data/localmeta/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv
deleted file mode 100644
index 441bf62..0000000
--- a/examples/test_case_data/localmeta/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv
+++ /dev/null
@@ -1,144 +0,0 @@
-48027,Coins & Paper Money:Paper Money: World:Asia:Japan,0,,,,10,Coins,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11116,Coins & Paper Money,3411,8008,48027,48027,48027,48027,Paper Money: World,Asia,Japan,Japan,Japan,Japan,262144,N,-999,Coins,5837,3069,48027,10,1,Collectibles
-164261,Jewelry & Watches:Fashion Jewelry:Earrings:Dangle & Chandelier:Sterling Silver (w/o Stone),0,,,,17,JewelrGemWatches,2008-08-12,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50647,50647,50647,50647,50647,Fashion Jewelry,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,2928,1873,50647,17,1,Fashion
-82494,BookComics & Mags:Comic Books:Modern (1980-now):Superhero:Sandman,15,,,,3,Books,2008-06-03,2010-09-13 07:05:54,USER_X,USER_Y,267,BookMagazines,63,63,63,63,63,63,NULL,Comic Books,Comic Books,Comic Books,Comic Books,Comic Books,262144,N,-999,Books,-999,-999,63,3,1,Media
-66767,Home & Lifestyle:Pet Supplies:Dog Supplies:Dog Apparel:Dresses,15,,,,16,Home & Garden,2008-06-03,2011-06-14 07:48:40,USER_X,USER_Y,11700,Home & Garden,1281,20742,1283,1283,1283,1283,NULL,Dogs,Other Supplies,Other Supplies,Other Supplies,Other Supplies,262144,N,-999,Home Improvement,-999,-999,1283,16,1,Home & Garden
-152801,Jewelry & Watches:Earrings:Studs:DiamonEnhanced Natural:PrincesSquare,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50647,50647,50647,50647,50647,NULL,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,2928,1873,50647,17,1,Fashion
-43398,Home & Garden:Food & Beverages:Cheese & Crackers,0,,,,36,Home & Living,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,11700,Home & Garden,14308,43398,43398,43398,43398,43398,NULL,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,Cheese & Crackers,262144,N,-999,Home Furnishing,-999,-999,43398,36,1,Home & Garden
-95173,Health & Beauty:Bath & Body:Bath SetKits:Avon,0,,,,37,Health & Beauty,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,26395,Health & Beauty,11838,67391,67391,67391,67391,67391,Bath & Body,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,Bath Sets & Kits,262144,N,-999,Health & Beauty,2919,1868,67391,37,1,Fashion
-158666,ToyHobbies:Action Figures:Anime & Manga,15,,,,28,Toys,2012-09-11,2012-09-11 20:27:22,USER_X,USER_Y,220,ToyHobbies,246,158666,158666,158666,158666,158666,Action Figures,Anime & Manga,Anime & Manga,Anime & Manga,Anime & Manga,Anime & Manga,262144,N,-999,Toys,-999,-999,158666,28,1,Collectibles
-12688,eBay Premier:Books & Manuscripts:Books: Other:Icollector,0,,2038,,57,Sothebys,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,2038,eBay Premier,1680,1681,12688,12688,12688,12688,Books & Manuscripts,Books: Other,Icollector,Icollector,Icollector,Icollector,60,N,,Sothebys,-999,-999,0,57,1,Lifestyle
-103324,Clothing & Accessories:Women:Shoes:Casual SneakerSkate:Sizes 10+,15,,,,9,Clothing & Accessories,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoeAccessories,3034,63889,63889,63889,63889,63889,Women's Shoes,Mixed Items,Mixed Items,Mixed Items,Mixed Items,Mixed Items,262144,N,-999,Clothing & Accessories,2441,1634,-99,9,1,Fashion
-108782,Vehicle Parts & Accessories:CaTruck Parts:Car Care & Cleaning:Air Fresheners,15,,,,8,Auto Parts,2012-09-11,2013-09-10 16:52:46,USER_X,USER_Y,131090,Vehicle Parts & Accessories,6030,72200,108782,108782,108782,108782,CaTruck Parts,Car Care & Cleaning,Air Fresheners,Air Fresheners,Air Fresheners,Air Fresheners,262144,N,-999,Auto - Parts,-999,-999,108782,8,1,Parts & Accessories
-80287,Computers & Networking:Software:Business & Productivity:Voice Recognition,0,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers/Tablets & Networking,18793,3768,3768,3768,3768,3768,Software,Office & Business,Office & Business,Office & Business,Office & Business,Office & Business,262144,N,-999,Computers,5893,3093,3768,12,1,Electronics
-140746,eBay Motors:Parts & Accessories:Vintage Car & Truck Parts:WheelTire& Hub Caps:Tires,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,10073,140744,140746,140746,140746,Parts & Accessories,Vintage Car & Truck Parts,WheelTire& Hub Caps,Tires,Tires,Tires,262144,N,-999,Auto - Parts,3517,1794,140746,8,1,Parts & Accessories
-87118,Sporting Goods:Paintball:Barrels:Intimidator,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,159043,16045,16046,16046,16046,16046,Outdoor Sports,Paintball,Barrels,Barrels,Barrels,Barrels,0,N,-999,Sporting Goods,2919,1868,16046,24,1,Lifestyle
-25147,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:Baseball-MLB:Philadelphia Phillies:Other Items,0,,888,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:51:44,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24410,24410,24410,24410,24410,Fan Apparel & Souvenirs,Baseball-MLB,Baseball-MLB,Baseball-MLB,Baseball-MLB,Baseball-MLB,262144,N,-999,Sports Memorabilia,2884,1845,24410,25,1,Collectibles
-170302,Crafts:Embroidery:Design CDs:Other Design Themes,15,,,,15,Crafts,2009-09-21,2010-09-13 07:13:34,USER_X,USER_Y,14339,Crafts,28141,41383,41383,41383,41383,41383,Embroidery,Design CDs,Design CDs,Design CDs,Design CDs,Design CDs,262144,N,-999,Hobbies & Crafts,-999,-999,41383,15,1,Home & Garden
-53064,Business & Industrial:Agriculture & Forestry:Antique Tractors & Equipment:Tractors:Massey HarriFerguson,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,177641,177642,177642,177642,177642,177642,Heavy Equipment,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,Antique & Vintage Farm Equip,262144,N,-999,Business (Office & Industrial),-999,-999,177642,4,1,Business & Industrial
-132939,Jewelry & Watches:Designer Brands:John Atencio,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,499,499,499,499,499,Fashion Jewelry,Other,Other,Other,Other,Other,262144,N,-999,JewelrGemWatches,2928,1873,499,17,1,Fashion
-113593,Mobiles & Phones:Mobile Phones:Samsung:Samsung D500,15,,,,40,Cell Phones & Portable Electro,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,15032,Phones,9355,9355,9355,9355,9355,9355,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,Mobile Phones,262144,N,-999,Cell Phones & Accessories,2246,1609,9355,40,1,Electronics
-34273,eBay Motors:Parts & Accessories:Motorcycle:Frame & Suspension:Frames,100,,,,8,Auto - Parts,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,6000,eBay Motors,6028,10063,34272,34273,34273,34273,Parts & Accessories,Motorcycle,Frame & Suspension,Frames,Frames,Frames,0,N,-999,Auto - Parts,-999,-999,0,8,1,Parts & Accessories
-106340,Home & Lifestyle:Tools & Building Materials:Hand Tools:Socket Sets,15,,,,16,Home & Garden,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,2032,29515,29515,29515,29515,29515,Gardening,Hand Tools,Hand Tools,Hand Tools,Hand Tools,Hand Tools,262144,N,-999,Home Improvement,-999,-999,29515,16,1,Home & Garden
-150265,Baby & Maternity:Baby Clothing & Shoes:Boys:Size 00:Outfits & Sets:Bonds,15,,,,2,Baby,2008-06-03,2011-04-05 08:25:25,USER_X,USER_Y,2984,Baby,3082,147317,147333,147333,147333,147333,Baby Clothing,Boys,OutfitSets,OutfitSets,OutfitSets,OutfitSets,0,N,-999,Baby,3090,1947,147333,2,1,Home & Garden
-24760,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:Hockey-NHL:San Jose Sharks:CapHats,0,,888,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:51:44,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24510,24510,24510,24510,24510,Fan Apparel & Souvenirs,Hockey-NHL,Hockey-NHL,Hockey-NHL,Hockey-NHL,Hockey-NHL,262144,N,-999,Sports Memorabilia,2884,1845,24510,25,1,Collectibles
-37831,Collectibles:Advertising:Merchandise & Memorabilia:Advertising-Print:1910-19,0,,,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,34,13623,37828,37831,37831,37831,Advertising,Merchandise & Memorabilia,Advertising-Print,1910-19,1910-19,1910-19,262144,N,-999,Collectibles,-999,-999,37831,11,1,Collectibles
-1120,Books:First Editions:Other,3,,267,Other,3,BookComics & Magazines,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,267,Books,2231,1120,1120,1120,1120,1120,First Editions,Other,Other,Other,Other,Other,0,N,-999,Books,,,0,3,1,Media
-43972,eBay Motors:Parts & Accessories:ATV Parts:Other,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,43962,43972,43972,43972,43972,Parts & Accessories,ATV Parts,Other,Other,Other,Other,262144,N,-999,Auto - Parts,5411,2941,43972,8,1,Parts & Accessories
-166013,Computers:Apple AccessorieParts:TV Tuners,15,,,,12,Computers,2008-09-09,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers,175673,3761,3761,3761,3761,3761,Computer Components & Parts,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,Video Capture & TV Tuner Cards,262144,N,-999,Computers,-999,-999,3761,12,1,Electronics
-15568,Baby:Baby Clothing:Unisex:Sleepwear,15,,,,2,Baby,2008-06-03,2011-04-05 08:25:25,USER_X,USER_Y,2984,Baby,3082,163397,163400,163400,163400,163400,Baby Clothing,Unisex,Sleepwear,Sleepwear,Sleepwear,Sleepwear,0,N,-999,Baby,3090,1947,163400,2,1,Home & Garden
-103178,Clothing & Accessories:Women:HandbagBags:Evening Bags,15,,,,9,Clothing & Accessories,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoeAccessories,169291,169291,169291,169291,169291,169291,Women's Bags,Women's Bags,Women's Bags,Women's Bags,Women's Bags,Women's Bags,262144,N,-999,Clothing & Accessories,2487,1098,-99,9,1,Fashion
-2023,Sporting Goods:Team Sports:Basketball:Other,0,,888,Sporting Goods,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,159049,21194,2023,2023,2023,2023,Team Sports,Basketball,Other,Other,Other,Other,262144,N,-999,Sporting Goods,-999,-999,2023,24,1,Lifestyle
-94847,Consumer Electronics:Car Electronics:Car Video:Car Video Monitors Only:Headrest Monitors:6-6.9 inches,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,3270,48604,48606,48606,48606,48606,Vehicle Electronics & GPS,Car Video,Car Monitors w/o Player,Car Monitors w/o Player,Car Monitors w/o Player,Car Monitors w/o Player,262144,N,-999,Consumer Electronics - Other,2920,1869,48606,13,1,Electronics
-15868,Real Estate:Land:Central U.S.:Illinois,0,,10542,,23,Real Estate,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,10542,Real Estate,15841,15841,15841,15841,15841,15841,Land,Land,Land,Land,Land,Land,0,N,-999,Real Estate,-999,-999,15841,23,1,Real Estate
-32876,Home & Garden:Home Improvement:Plumbing & Fixtures:Toilet Paper Holders-Mounted,0,,,,36,Home & Living,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,159907,20601,32876,32876,32876,32876,Home Improvement,Plumbing & Fixtures,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,Toilet Paper Holders-Mounted,262144,N,-999,Home Furnishing,-999,-999,32876,36,1,Home & Garden
-62179,Sporting Goods:Skiing & Snowboarding:Apparel:Ski Suits:Women,0,,,,9,Apparel,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,137085,137085,137085,137085,137085,Women's Clothing,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,0,N,-999,Clothing & Accessories,2919,1868,137085,9,1,Fashion
-45333,ClothinShoes & Accessories:Women's Shoes:Flats & Oxfords,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,45333,45333,45333,45333,45333,Women's Shoes,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,Flats & Oxfords,262144,N,-999,Clothing & Accessories,-999,-999,45333,9,1,Fashion
-50677,Jewelry & Watches:Fashion Jewelry:Pins & Brooches,0,,,,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,50677,50677,50677,50677,50677,Fashion Jewelry,Pins & Brooches,Pins & Brooches,Pins & Brooches,Pins & Brooches,Pins & Brooches,262144,N,-999,JewelrGemWatches,-999,-999,50677,17,1,Fashion
-33038,Musical Instruments:Guitars:Electric:ESP,15,,,,30,Musical Instruments,2008-06-03,2010-06-14 07:48:40,USER_X,USER_Y,619,Musical Instruments,166070,33034,33034,33034,33034,33034,Instruments,Guitars (Electric),Guitars (Electric),Guitars (Electric),Guitars (Electric),Guitars (Electric),262144,N,-999,Musical Instruments,-999,-999,33034,30,1,Lifestyle
-156614,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:NASCAR:Revell:Dale Jarrett,0,,,,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,171127,762,762,762,762,Diecast & Toy Vehicles,Cars: RacinNASCAR,NASCAR,NASCAR,NASCAR,NASCAR,262144,N,-999,Toys,1740,1380,762,28,1,Collectibles
-106246,Health & Beauty:Hair Care:Conditioner:Biolage,0,,,,37,Health & Beauty,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,26395,Health & Beauty,11854,177661,177661,177661,177661,177661,Hair Care & Styling,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,Shampoo & Conditioning,262144,N,-999,Health & Beauty,2919,1868,177661,37,1,Fashion
-20865,Sporting Goods:Soccer:Apparel & Footwear:Jerseys:Other,0,,888,,9,Apparel,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,137084,137084,137084,137084,137084,Men's Clothing,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,Athletic Apparel,0,N,-999,Clothing & Accessories,2919,1868,137084,9,1,Fashion
-15115,Entertainment:Video Games:Games:Sega Dreamcast:Sports,0,,293,,32,Video Games,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Video Games,Video Games,Video Games,Video Games,Video Games,Video Games,0,N,-999,Video Games,-999,-999,139973,32,1,Electronics
-3838,JewelrGemstones:Jewelry:Charms:Plated,0,,281,Other,17,JewelrGemWatches,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,281,Jewelry & Watches,10968,140944,140944,140944,140944,140944,Fashion Jewelry,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,Charms & Charm Bracelets,0,N,-999,JewelrGemWatches,-999,-999,140944,17,1,Fashion
-759,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:Franklin Mint:Modern (1970-Now),0,,220,Other,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,19036,223,223,223,223,Diecast & Toy Vehicles,CarTrucks & Vans,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,262144,N,-999,Toys,1740,1380,223,28,1,Collectibles
-61323,Consumer Electronics:A/V Accessories & Cables:Remote Controls:Universal Remotes,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,32852,14961,61312,61312,61312,61312,TVideo & Home Audio,TVideo & Audio Accessories,Remote Controls,Remote Controls,Remote Controls,Remote Controls,262144,N,-999,Consumer Electronics - Other,5344,2248,61312,13,1,Electronics
-121153,Baby:Nursery Decor:Night Lights,0,,,,2,Baby,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,2984,Baby,66697,121153,121153,121153,121153,121153,Nursery Decor,Night Lights,Night Lights,Night Lights,Night Lights,Night Lights,262144,N,-999,Baby,-999,-999,121153,2,1,Home & Garden
-88750,Consumer Electronics:Car Electronics:RadaLaser Detectors:Whistler:17174 Series,0,,,,13,Consumer Electronics - Other,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,293,Consumer Electronics,3270,14935,14935,14935,14935,14935,Vehicle Electronics & GPS,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,Radar & Laser Detectors,262144,N,-999,Consumer Electronics - Other,2920,1869,14935,13,1,Electronics
-161567,Computers:Laptop Accessories:Batteries:Hewlett Packard,15,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers,31530,14295,14295,14295,14295,14295,Laptop & Desktop Accessories,Laptop Batteries,Laptop Batteries,Laptop Batteries,Laptop Batteries,Laptop Batteries,262144,N,-999,Computers,3699,2194,14295,12,1,Electronics
-113802,Lots More...:Metaphysical:Herbs,15,,,,99,Everything Else,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,99,Lots More...,19266,113802,113802,113802,113802,113802,Metaphysical,Herbs,Herbs,Herbs,Herbs,Herbs,262144,N,-999,Everything Else,-999,-999,113802,99,1,Lifestyle
-15808,Clothing & Accessories:Women:Plus Sizes:Tops:Other Tops,15,,,,9,Clothing & Accessories,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoeAccessories,15724,53159,53159,53159,53159,53159,Women's Clothing,Tops & Blouses,Tops & Blouses,Tops & Blouses,Tops & Blouses,Tops & Blouses,262144,N,-999,Clothing & Accessories,2264,1551,53159,9,1,Fashion
-174053,Vehicle Parts & Accessories:Car Parts:External & Body Parts:SunrooConvertible & Hardtop,3,,,,8,Auto - Parts,2011-08-23,2013-04-28 20:37:19,USER_X,USER_Y,131090,Vehicle Parts & Accessories,9884,14768,174053,174053,174053,174053,Car Parts,External & Body Parts,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,SunrooConvertible & Hardtop,262144,N,-999,Auto - Parts,4075,2390,174053,8,1,Parts & Accessories
-2635,Toys & Hobbies:Toy Soldiers:1970-Now:Plastic,0,,220,Other,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,2631,2638,2638,2638,2638,2638,Toy Soldiers,1970-Now,1970-Now,1970-Now,1970-Now,1970-Now,262144,N,-999,Toys,2919,1868,2638,28,1,Collectibles
-1161,DVFilm & TV:Other Formats:Videos: NTSC (US):ActioAdventure,3,,267,Other,18,DVDFilm & TV,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11232,DVFilm & TV,380,1151,1161,1161,1161,1161,Other Formats,Videos: NTSC (US),ActioAdventure,ActioAdventure,ActioAdventure,ActioAdventure,0,N,-999,DVDs & Movies,-999,-999,0,18,1,Media
-64076,Computers & Networking:Networking:Switches:3Com:SuperStack,0,,,,12,Computers,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,58058,Computers/Tablets & Networking,175698,175706,51268,51268,51268,51268,Enterprise NetworkinServers,Switches & Hubs,Network Switches,Network Switches,Network Switches,Network Switches,262144,N,-999,Computers,2919,1868,51268,12,1,Electronics
-33977,Crafts:Scrapbooking:Albums:12x12 Albums,15,,,,15,Crafts,2008-06-03,2010-06-14 07:56:25,USER_X,USER_Y,14339,Crafts,31253,33875,33875,33875,33875,33875,Scrapbooking,Albums,Albums,Albums,Albums,Albums,262144,N,-999,Hobbies & Crafts,-999,-999,33875,15,1,Home & Garden
-31673,Sports:Memorabilia:Racing-NASCAR:Plaques,0,,,,25,Sports Memorabilia,2008-06-03,2010-02-08 10:48:39,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,46156,46156,46156,46156,46156,Fan Apparel & Souvenirs,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,Racing-NASCAR,0,N,-999,Sports Memorabilia,-999,-999,46156,25,1,Collectibles
-174106,Vehicle Parts & Accessories:Car Parts:Transmission & Drivetrain:Sensors,3,,,,8,Auto - Parts,2011-08-23,2013-04-28 20:37:19,USER_X,USER_Y,131090,Vehicle Parts & Accessories,9884,33726,174106,174106,174106,174106,Car Parts,Transmission & Drivetrain,Sensors,Sensors,Sensors,Sensors,262144,N,-999,Auto - Parts,4075,2390,174106,8,1,Parts & Accessories
-26249,Business & Industrial:Printing & Graphic Arts:Commercial Printing Presses:Sheet-fed Offset Presses,0,,9428,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,26238,26247,26249,26249,26249,26249,Printing & Graphic Arts,Commercial Printing Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,Sheet-fed Offset Presses,262144,N,-999,Business (Office & Industrial),-999,-999,26249,4,1,Business & Industrial
-159184,Sporting Goods:Winter Sports:Snowboarding:Stomp Pads,0,,,,24,Sporting Goods,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,888,Sporting Goods,36259,21247,159184,159184,159184,159184,Winter Sports,Snowboarding,Stomp Pads,Stomp Pads,Stomp Pads,Stomp Pads,262144,N,-999,Sporting Goods,-999,-999,159184,24,1,Lifestyle
-10058,Tickets & Travel:Tickets:Sporting Events:Wrestling,3,,11730,,27,Tickets,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,1305,Events Tickets,1306,1306,1306,1306,1306,1306,Other Tickets,Other Tickets,Other Tickets,Other Tickets,Other Tickets,Other Tickets,262144,N,-999,Tickets,2937,1879,1306,27,1,Lifestyle
-48904,ClothinShoes & Accessories:Vintage:Women's Vintage Clothing:1977-89 (PunNew Wav80s):Sweaters,0,,,,9,Apparel,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoes & Accessories,175759,175762,52451,48904,48904,48904,Vintage,Women's Vintage Clothing,1977-89 (PunNew Wav80s),Sweaters,Sweaters,Sweaters,262144,N,-999,Clothing & Accessories,-999,-999,48904,9,1,Fashion
-145970,Toys & Hobbies:ModelKits:Automotive:Vintage:Plymouth,0,,,,28,Toys,2008-06-03,2008-10-08 07:18:40,USER_X,USER_Y,220,Toys & Hobbies,1188,2580,2585,2585,2585,2585,Models & Kits,Automotive,Vintage,Vintage,Vintage,Vintage,262144,N,-999,Toys,2974,1905,2585,28,1,Collectibles
-963,ClothinShoes & Accessories:Vintage:Clothing (Pre-1980):Women:ClothinShoes & Accessories:Shoes,0,,1,Other,9,Apparel,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11450,ClothinShoes & Accessories,175759,74976,48561,48561,48561,48561,Vintage,Women's Vintage Shoes,Other,Other,Other,Other,0,N,-999,Clothing & Accessories,-999,-999,48561,9,1,Fashion
-118687,Health & Beauty:Fragrances:Women's Fragrances:Benetton,3,,,,37,Health & Beauty,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,26395,Health & Beauty,26396,11848,11848,11848,11848,11848,Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,Women's Fragrances,262144,N,-999,Health & Beauty,6778,3189,11848,37,1,Fashion
-20886,Toys & Hobbies:Diecast Toy Vehicles:CarTrucks-Diecast:NASCAR:Action/ Motorsports Authentic:Darrell Waltrip,0,,220,,28,Toys,2008-06-03,2010-03-22 10:34:30,USER_X,USER_Y,220,Toys & Hobbies,222,171127,762,762,762,762,Diecast & Toy Vehicles,Cars: RacinNASCAR,NASCAR,NASCAR,NASCAR,NASCAR,262144,N,-999,Toys,1740,1380,762,28,1,Collectibles
-148324,Mobiles & Phones:Mobile Accessories:Covers / Faceplates:Sony Ericsson:W900i,15,,,,31,Phones,2008-06-03,2012-09-18 00:08:03,USER_X,USER_XIANZHU,15032,Phones,9394,20349,20349,20349,20349,20349,Mobile Accessories,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,CaseCoverSkins,262144,N,-999,Telecomm,-999,-999,20349,31,1,Electronics
-139255,JewellerGemWatches:Fine Jewellery:Earrings:Gemstone - Created:Lapis,15,,,,17,JewelrGemWatches,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,281,Jewellery & Watches,4196,10985,10985,10985,10985,10985,Fine Jewellery,Earrings,Earrings,Earrings,Earrings,Earrings,262144,N,-999,JewelrGemWatches,-999,-999,-99,17,1,Fashion
-20213,Collectibles:Postcards:US StateCities & Towns:Maine,0,,1,,11,Collectibles,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,1,Collectibles,914,20193,20213,20213,20213,20213,Postcards,US StateCities & Towns,Maine,Maine,Maine,Maine,262144,N,-999,Collectibles,-999,-999,20213,11,1,Collectibles
-32996,Movies:Television Memorabilia:Clippings,15,,,,42,Entertainment Memorabilia,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11232,Movies,1424,32996,32996,32996,32996,32996,Television Memorabilia,Clippings,Clippings,Clippings,Clippings,Clippings,262144,N,-999,Entertainment Memorabilia,-999,-999,32996,42,1,Collectibles
-99985,Collectibles:Trading Cards:Sci-FFantasy:Star Wars:Sets:Other,0,,,,11,Collectibles,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,1,Collectibles,868,37897,100899,99980,99985,99985,Trading Cards,Sci-FFantasy,Star Wars,Sets,Other,Other,262144,N,-999,Collectibles,-999,-999,99985,11,1,Collectibles
-67703,Jewellery & Watches:Jewellery Boxes & Supplies:Jewellery Display,3,,,,17,Jewellery & Watches,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,281,Jewellery & Watches,10321,67703,67703,67703,67703,67703,Jewellery Boxes & Supplies,Jewellery Display,Jewellery Display,Jewellery Display,Jewellery Display,Jewellery Display,262144,N,-999,JewelrGemWatches,4435,2261,67703,17,1,Fashion
-164,Computers/Tablets & Networking:Computer Components & Parts:CPUProcessors,0,,160,Computer Hardware,12,Computers,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,164,164,164,164,164,Computer Components & Parts,CPUProcessors,CPUProcessors,CPUProcessors,CPUProcessors,CPUProcessors,262144,N,-999,Computers,5730,2011,164,12,1,Electronics
-279,BookMagazines:Children's Books,15,,267,Other,3,Books,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,267,BookMagazines,1093,1093,1093,1093,1093,1093,Children's Books,Children's Books,Children's Books,Children's Books,Children's Books,Children's Books,262144,N,-999,Books,1415,1128,1093,3,1,Media
-65,Collectibles:Comics:Platinum Age (1897-1937),0,,1,Other,11,Collectibles,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,1,Collectibles,63,65,65,65,65,65,Comics,Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),Platinum Age (1897-1937),262144,N,-999,Collectibles,6014,2967,65,11,1,Collectibles
-24541,Sports MeCards & Fan Shop:Fan Apparel & Souvenirs:College-NCAA,0,,,,25,Sports Memorabilia,2010-02-08,2013-07-16 08:23:09,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,24409,24541,24541,24541,24541,24541,Fan Apparel & Souvenirs,College-NCAA,College-NCAA,College-NCAA,College-NCAA,College-NCAA,262144,N,-999,Sports Memorabilia,7721,2482,24541,25,1,Collectibles
-23446,Mode & Accessoires:Chaussures de femme:Sandales & Sandalettes:Taille 37-38,23,,,,9,Vtements et Accessoires,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,11450,Mode & Accessoires,35107,23418,23446,23446,23446,23446,Chaussures de femme,Sandales & Sandalettes,Taille 37-38,Taille 37-38,Taille 37-38,Taille 37-38,262144,N,-999,Clothing & Accessories,2438,1634,23446,9,1,Fashion
-130,Collectibles:Transportation:Railroadiana & Trains:Other,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,417,1444,130,130,130,130,Transportation,Railroadiana & Trains,Other,Other,Other,Other,262144,N,-999,Collectibles,-999,-999,130,11,1,Collectibles
-216,Sports MeCards & Fan Shop:Cards:Hockey,0,,11800,Other,25,Sports Memorabilia,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,64482,Sports MeCards & Fan Shop,212,216,216,216,216,216,Cards,Hockey,Hockey,Hockey,Hockey,Hockey,262144,N,-999,Sports Memorabilia,3636,2152,216,25,1,Collectibles
-10866,Collectibles:Animals:Farm & Countryside:Pigs,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,1335,45602,10866,10866,10866,10866,Animals,Farm & Countryside,Pigs,Pigs,Pigs,Pigs,262144,N,-999,Collectibles,-999,-999,10866,11,1,Collectibles
-57013,Business & Industrial:MRO & Industrial Supply:Pumps & Plumbing:Pumps:Pump Accessories & Parts,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,1266,109619,42928,57013,57013,57013,MRO & Industrial Supply,Pumps & Plumbing,Pumps,Pump Accessories & Parts,Pump Accessories & Parts,Pump Accessories & Parts,262144,N,-999,Business (Office & Industrial),-999,-999,57013,4,1,Business & Industrial
-15687,ClothinShoes & Accessories:Men's Clothing:T-Shirts,0,,11450,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,15687,15687,15687,15687,15687,Men's Clothing,T-Shirts,T-Shirts,T-Shirts,T-Shirts,T-Shirts,262144,N,2515,Clothing & Accessories,7692,1115,15687,9,1,Fashion
-80135,Computers/Tablets & Networking:DriveStorage & Blank Media:Blank Media & Accessories:CDVD & Blu-ray Discs,0,,,,12,Computers,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,165,80133,80135,80135,80135,80135,DriveStorage & Blank Media,Blank Media & Accessories,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,CDVD & Blu-ray Discs,262144,N,-999,Computers,3374,2018,80135,12,1,Electronics
-31519,Computers/Tablets & Networking:Laptop & Desktop Accessories:Laptop Cases & Bags,0,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,31530,31519,31519,31519,31519,31519,Laptop & Desktop Accessories,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,262144,N,-999,Computers,6404,3189,31519,12,1,Electronics
-175750,Home & Garden:Bedding:Blankets & Throws,0,,,,36,Home & Living,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,11700,Home & Garden,20444,175750,175750,175750,175750,175750,Bedding,Blankets & Throws,Blankets & Throws,Blankets & Throws,Blankets & Throws,Blankets & Throws,262144,N,-999,Home Furnishing,-999,-999,175750,36,1,Home & Garden
-63864,ClotheShoes & Accessories:Women's Clothing:Skirts,3,,,,9,Clothing & Accessories,2009-06-15,2012-06-19 21:15:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,15724,63864,63864,63864,63864,63864,Women's Clothing,Skirts,Skirts,Skirts,Skirts,Skirts,262144,N,-999,Clothing & Accessories,6334,1551,63864,9,1,Fashion
-13836,Collectibles:Decorative Collectibles:Spoons,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,13836,13836,13836,13836,13836,Decorative Collectibles,Spoons,Spoons,Spoons,Spoons,Spoons,262144,N,-999,Collectibles,-999,-999,13836,11,1,Collectibles
-63861,ClothinShoes & Accessories:Women's Clothing:Dresses,0,,,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,63861,63861,63861,63861,63861,Women's Clothing,Dresses,Dresses,Dresses,Dresses,Dresses,262144,N,-999,Clothing & Accessories,1263,1112,63861,9,1,Fashion
-150047,Crafts:Jewellery Making:Findings:Bails,3,,,,15,Hobbies & Crafts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,14339,Crafts,150045,150046,150047,150047,150047,150047,Jewellery Making,Findings,Bails,Bails,Bails,Bails,262144,N,-999,Hobbies & Crafts,4618,2740,150047,15,1,Home & Garden
-95672,ClotheShoes & Accessories:Women's Shoes:Trainers,3,,,,9,Clothing & Accessories,2011-04-05,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,3034,95672,95672,95672,95672,95672,Women's Shoes,Trainers,Trainers,Trainers,Trainers,Trainers,262144,N,-999,Clothing & Accessories,2314,1634,95672,9,1,Fashion
-57990,ClotheShoes & Accessories:Men's Clothing:Casual Shirts & Tops,3,,,,9,Clothing & Accessories,2009-06-15,2012-06-19 21:15:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,1059,57990,57990,57990,57990,57990,Men's Clothing,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,Casual Shirts & Tops,262144,N,-999,Clothing & Accessories,6331,1623,57990,9,1,Fashion
-75708,Toys & Games:Action Figures:TMovies & Video Games,3,,,,28,Toys & Games,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,220,Toys & Games,246,75708,75708,75708,75708,75708,Action Figures,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,TMovies & Video Games,262144,N,-999,Toys,-999,-999,75708,28,1,Collectibles
-4943,Toys & Hobbies:Diecast & Toy Vehicles:CarTrucks & Vans:Plastic:Vintage Manufacture (Pre-1970),0,,220,Other,28,Toys,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,220,Toys & Hobbies,222,19036,19060,4943,4943,4943,Diecast & Toy Vehicles,CarTrucks & Vans,Plastic,Vintage Manufacture (Pre-1970),Vintage Manufacture (Pre-1970),Vintage Manufacture (Pre-1970),262144,N,-999,Toys,1740,1380,4943,28,1,Collectibles
-40059,Mobile Phones & Communication:Radio Communication Equipment:Parts & Accessories:Other Parts & Accessories,3,,,,33,Consumer Electronics - Audio,2012-05-01,2012-05-01 08:57:38,USER_X,USER_Y,15032,Mobile Phones & Communication,1500,175734,40059,40059,40059,40059,Radio Communication Equipment,Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,Other Parts & Accessories,262144,N,-999,Consumer Electronics - Audio,-999,-999,40059,33,1,Electronics
-43479,Cameras & Photo:Film Photography:Other,0,,,,21,Photo,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,625,Cameras & Photo,69323,43479,43479,43479,43479,43479,Film Photography,Other,Other,Other,Other,Other,262144,N,-999,Photo,-999,-999,43479,21,1,Electronics
-11848,Health & Beauty:Fragrances:Women,0,,,,37,Health & Beauty,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,26395,Health & Beauty,26396,11848,11848,11848,11848,11848,Fragrances,Women,Women,Women,Women,Women,262144,N,-999,Health & Beauty,5884,3087,11848,37,1,Fashion
-165888,Jewelry & Watches:Vintage & Antique Jewelry:Costume:RetrVintage 1930s-1980s:PinBrooches,0,,,,17,JewelrGemWatches,2008-09-09,2009-01-12 07:05:17,USER_X,USER_Y,281,Jewelry & Watches,48579,500,58559,165888,165888,165888,Vintage & Antique Jewelry,Costume,RetrVintage 1930s-1980s,PinBrooches,PinBrooches,PinBrooches,262144,N,-999,JewelrGemWatches,-999,-999,165888,17,1,Fashion
-26262,Collectibles:Advertising:Food & Beverage:Distillery:Bacardi,0,,1,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,34,13587,820,26262,26262,26262,Advertising,Food & Beverage,Distillery,Bacardi,Bacardi,Bacardi,262144,N,-999,Collectibles,-999,-999,26262,11,1,Collectibles
-67698,Business & Industrial:Retail & Services:Jewellery Packaging & Display:Cases & Displays:Bracelet,2,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11890,67694,67697,67698,67698,67698,Retail & Services,Jewellery Packaging & Display,Cases & Displays,Bracelet,Bracelet,Bracelet,262144,N,-999,Business (Office & Industrial),-999,-999,67698,4,1,Business & Industrial
-57990,ClothinShoes & Accessories:Men's Clothing:Casual Shirts,0,,,,9,Apparel,2008-06-03,2013-09-10 16:52:46,USER_X,USER_Y,11450,ClothinShoes & Accessories,1059,57990,57990,57990,57990,57990,Men's Clothing,Casual Shirts,Casual Shirts,Casual Shirts,Casual Shirts,Casual Shirts,262144,N,-999,Clothing & Accessories,-999,-999,57990,9,1,Fashion
-67698,Business & Industrial:Retail & Services:Jewelry Packaging & Display:Cases & Displays:Bracelet,0,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11890,67694,67697,67698,67698,67698,Retail & Services,Jewelry Packaging & Display,Cases & Displays,Bracelet,Bracelet,Bracelet,262144,N,-999,Business (Office & Industrial),-999,-999,67698,4,1,Business & Industrial
-46575,Business & Industrial:Light Equipment & Tools:Air Tools:Air Tool Accessories,0,,,,4,Business (Office & Industrial),2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,12576,Business & Industrial,61573,46573,46575,46575,46575,46575,Light Equipment & Tools,Air Tools,Air Tool Accessories,Air Tool Accessories,Air Tool Accessories,Air Tool Accessories,262144,N,-999,Business (Office & Industrial),-999,-999,46575,4,1,Business & Industrial
-41940,Business & Industrial:Manufacturing & Metalworking:Metalworking Tooling:Toolholding:Indexable Inserts,0,,,,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,11804,92084,45017,41940,41940,41940,Manufacturing & Metalworking,Metalworking Tooling,Toolholding,Indexable Inserts,Indexable Inserts,Indexable Inserts,262144,N,-999,Business (Office & Industrial),-999,-999,41940,4,1,Business & Industrial
-51582,ClothinShoes & Accessories:Kids' ClothinShoes & Accs:Girls' Clothing (Sizes 4 & Up):Sweaters,0,,,,9,Apparel,2012-05-01,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,171146,11462,51582,51582,51582,51582,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Sweaters,Sweaters,Sweaters,Sweaters,262144,N,-999,Clothing & Accessories,-999,-999,51582,9,1,Fashion
-80053,Computers/Tablets & Networking:MonitorProjectors & Accs:Monitors,0,,,,12,Computers,2009-09-21,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,162497,80053,80053,80053,80053,80053,MonitorProjectors & Accs,Monitors,Monitors,Monitors,Monitors,Monitors,262144,N,-999,Computers,2982,1911,80053,12,1,Electronics
-31387,Jewellery & Watches:Watches:Wristwatches,3,,,,17,Jewellery & Watches,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,281,Jewellery & Watches,14324,31387,31387,31387,31387,31387,Watches,Wristwatches,Wristwatches,Wristwatches,Wristwatches,Wristwatches,262144,N,-999,JewelrGemWatches,7489,32,31387,17,1,Fashion
-20485,Home & Garden:Inside the Home:Furniture:Dining Room:Other,0,,11700,,36,Home & Living,2008-06-03,2012-05-01 08:54:26,USER_X,USER_Y,11700,Home & Garden,3197,175752,175752,175752,175752,175752,Furniture,Other,Other,Other,Other,Other,262144,N,-999,Home Furnishing,2919,1868,175752,36,1,Home & Garden
-44079,Sporting Goods:Exercise & Fitness:GyWorkout & Yoga:Fitness Equipment:Exercise Mats,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,15273,158932,28064,44079,44079,44079,Exercise & Fitness,GyWorkout & Yoga,Fitness Equipment,Exercise Mats,Exercise Mats,Exercise Mats,262144,N,-999,Sporting Goods,-999,-999,44079,24,1,Lifestyle
-175750,HomFurniture & DIY:Bedding:Blankets,3,,,,16,Home,2012-05-01,2012-09-11 20:26:04,USER_X,USER_Y,11700,HomFurniture & DIY,20444,175750,175750,175750,175750,175750,Bedding,Blankets,Blankets,Blankets,Blankets,Blankets,262144,N,-999,Home Improvement,-999,-999,175750,16,1,Home & Garden
-16509,Toys & Hobbies:Model Railroads & Trains:S Scale:Other,0,,220,,28,Toys,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,220,Toys & Hobbies,479,19143,16509,16509,16509,16509,Model Railroads & Trains,S Scale,Other,Other,Other,Other,262144,N,-999,Toys,-999,-999,16509,28,1,Collectibles
-139973,Video Games & Consoles:Games,3,,,,32,PC & Video Gaming,2011-04-05,2012-09-11 20:26:04,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Games,Games,Games,Games,Games,Games,262144,N,-999,Video Games,1655,1135,139973,32,1,Electronics
-170083,Computers/Tablets & Networking:Computer Components & Parts:Memory (RAM),3,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,170083,170083,170083,170083,170083,Computer Components & Parts,Memory (RAM),Memory (RAM),Memory (RAM),Memory (RAM),Memory (RAM),262144,N,-999,Computers,3369,2042,170083,12,1,Electronics
-16145,Computers/Tablets & Networking:Computer Components & Parts:Other Components & Parts,3,,160,,12,Computers,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,16145,16145,16145,16145,16145,Computer Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,Other Components & Parts,262144,N,-999,Computers,6976,3189,16145,12,1,Electronics
-158798,Toys & Hobbies:Vintage & Antique Toys:Spinning Tops,0,,,,28,Toys,2008-06-03,2008-09-09 22:08:47,USER_X,USER_Y,220,Toys & Hobbies,717,158798,158798,158798,158798,158798,Vintage & Antique Toys,Spinning Tops,Spinning Tops,Spinning Tops,Spinning Tops,Spinning Tops,262144,N,-999,Toys,-999,-999,158798,28,1,Collectibles
-20485,CasArredamento e Bricolage:Cucina:Altro per cucina,101,,,,36,Mobili per la casa,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,11700,CasArredamento e Bricolage,20625,20485,20485,20485,20485,20485,Cucina,Altro per cucina,Altro per cucina,Altro per cucina,Altro per cucina,Altro per cucina,0,N,-999,Home Furnishing,-999,-999,20485,36,1,Home & Garden
-100847,Half Books,0,,,,3,Books,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,100847,Half Books,100847,100847,100847,100847,100847,100847,Half Books,Half Books,Half Books,Half Books,Half Books,Half Books,0,N,-999,Books,-999,-999,100847,3,2,Media
-156356,Collectibles:Postcards:BuildingArchitecture,0,,,,11,Collectibles,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,1,Collectibles,914,156356,156356,156356,156356,156356,Postcards,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,BuildingArchitecture,262144,N,-999,Collectibles,-999,-999,156356,11,1,Collectibles
-60606,Collectables:Badges/ Patches:Golly Badges:Robertsons:Modern (Post-1990),3,,,,11,Collectables,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1,Collectables,32615,98793,106160,60606,60606,60606,Badges/ Patches,Golly Badges,Robertsons,Modern (Post-1990),Modern (Post-1990),Modern (Post-1990),262144,N,-999,Collectibles,4390,2640,60606,11,1,Collectibles
-36250,Sporting Goods:Outdoor Sports:Hunting:Decoys:Duck,0,,,,24,Sporting Goods,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,888,Sporting Goods,7301,36249,36249,36249,36249,36249,Hunting,Decoys,Decoys,Decoys,Decoys,Decoys,262144,N,-999,Sporting Goods,-999,-999,36249,24,1,Lifestyle
-50508,Cameras & Photo:Camera & Photo Accessories:LCD Hoods,0,,,,21,Photo,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,625,Cameras & Photo,15200,50508,50508,50508,50508,50508,Camera & Photo Accessories,LCD Hoods,LCD Hoods,LCD Hoods,LCD Hoods,LCD Hoods,262144,N,-999,Photo,6256,3170,50508,21,1,Electronics
-9426,Mobile Phones & Communication:Home Phones & Accessories:Phone Accessories:Other Phone Accessories,3,,,,31,Mobile & Home Phones,2011-04-05,2012-05-01 08:54:26,USER_X,USER_Y,15032,Mobile Phones & Communication,3286,9422,9426,9426,9426,9426,Home Phones & Accessories,Phone Accessories,Other Phone Accessories,Other Phone Accessories,Other Phone Accessories,Other Phone Accessories,262144,N,-999,Telecomm,-999,-999,9426,31,1,Electronics
-45238,Abbigliamento e accessori:Donna: Accessori:SciarpFoulard e Scialli,101,,,,9,Vestiti ed Accessori,2012-09-11,2012-09-11 20:27:22,USER_X,USER_Y,11450,Abbigliamento e accessori,4251,45238,45238,45238,45238,45238,Donna: Accessori,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,SciarpFoulard e Scialli,0,N,-999,Clothing & Accessories,-999,-999,45238,9,1,Fashion
-75665,Home & Garden:YarGarden & Outdoor Living:Gardening Supplies:Garden Tools & Equipment:Composting,0,,,,16,Home Improvement,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,159912,2032,29515,75665,75665,75665,YarGarden & Outdoor Living,Gardening Supplies,Garden Tools & Equipment,Composting,Composting,Composting,262144,N,-999,Home Improvement,-999,-999,75665,16,1,Home & Garden
-57784,ClothinShoes & Accessories:Baby & Toddler Clothing:Boys' Clothing (Newborn-5T):One-Pieces,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3082,147317,57784,57784,57784,57784,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),One-Pieces,One-Pieces,One-Pieces,One-Pieces,262144,N,-999,Clothing & Accessories,-999,-999,57784,9,1,Fashion
-6762,eBay Motors:Parts & Accessories:Manuals:Manuals:Other Makes,0,,6001,Other,-999,Unknown,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,-999,Unknown,-999,-999,-999,-999,-999,-999,Unknown,Unknown,Unknown,Unknown,Unknown,Unknown,0,N,-999,Unknown,-999,-999,-999,-999,1,Unknown
-60340,Entertainment Memorabilia:Movie Memorabilia:Pressbooks:1970-79,0,,,,42,Entertainment Memorabilia,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,45100,Entertainment Memorabilia,196,18830,60340,60340,60340,60340,Movie Memorabilia,Pressbooks,1970-79,1970-79,1970-79,1970-79,262144,N,-999,Entertainment Memorabilia,-999,-999,60340,42,1,Collectibles
-35570,eBay Motors:Parts & Accessories:Motorcycle Parts:DecalEmblems,100,,,,8,Auto - Parts,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,6000,eBay Motors,6028,10063,35570,35570,35570,35570,Parts & Accessories,Motorcycle Parts,DecalEmblems,DecalEmblems,DecalEmblems,DecalEmblems,262144,N,-999,Auto - Parts,5407,2940,35570,8,1,Parts & Accessories
-11554,ClothinShoes & Accessories:Women's Clothing:Jeans,0,,11450,,9,Apparel,2008-06-03,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,11554,11554,11554,11554,11554,Women's Clothing,Jeans,Jeans,Jeans,Jeans,Jeans,262144,N,-999,Clothing & Accessories,7720,1113,11554,9,1,Fashion
-1357,Collectibles:Decorative Collectibles:Decorative Collectible Brands:Hamilton Collector Plates,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,156286,1357,1357,1357,1357,Decorative Collectibles,Decorative Collectible Brands,Hamilton Collector Plates,Hamilton Collector Plates,Hamilton Collector Plates,Hamilton Collector Plates,262144,N,-999,Collectibles,-999,-999,1357,11,1,Collectibles
-13987,Collectibles:Paper:Booklets:1920-39,0,,1,,11,Collectibles,2008-06-03,2008-06-03 13:36:00,USER_X,USER_Y,1,Collectibles,124,13985,13987,13987,13987,13987,Paper,Booklets,1920-39,1920-39,1920-39,1920-39,262144,N,-999,Collectibles,-999,-999,13987,11,1,Collectibles
-63861,ClotheShoes & Accessories:Women's Clothing:Dresses,3,,,,9,Clothing & Accessories,2009-06-15,2013-07-16 08:23:09,USER_X,USER_Y,11450,ClotheShoes & Accessories,15724,63861,63861,63861,63861,63861,Women's Clothing,Dresses,Dresses,Dresses,Dresses,Dresses,262144,N,-999,Clothing & Accessories,-999,-999,63861,9,1,Fashion
-314,ClothinShoes & Accessories:Womens' Clothing:Other,211,,,,9,ClothinShoes & Accessories,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,314,314,314,314,314,Womens' Clothing,Other,Other,Other,Other,Other,0,N,-999,Clothing & Accessories,-999,-999,314,9,1,Fashion
-139973,Video Games & Consoles:Video Games,0,,,,32,Video Games,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,1249,Video Games & Consoles,139973,139973,139973,139973,139973,139973,Video Games,Video Games,Video Games,Video Games,Video Games,Video Games,262144,N,-999,Video Games,1453,1135,139973,32,1,Electronics
-30059,Cameras & Photography:Lenses & Filters:Lens AdapterMounts & Tubes,3,,,,21,Photography,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,625,Cameras & Photography,78997,30059,30059,30059,30059,30059,Lenses & Filters,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,Lens AdapterMounts & Tubes,262144,N,-999,Photo,4379,2630,30059,21,1,Electronics
-16145,Computers/Tablets & Networking:Computer Components & Parts:Other,0,,,,12,Computers,2008-06-03,2013-02-20 23:11:43,USER_X,USER_Y,58058,Computers/Tablets & Networking,175673,16145,16145,16145,16145,16145,Computer Components & Parts,Other,Other,Other,Other,Other,262144,N,-999,Computers,7312,3189,16145,12,1,Electronics
-95672,ClothinShoes & Accessories:Women's Shoes:Athletic,0,,,,9,Apparel,2011-04-05,2013-02-20 23:11:43,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,95672,95672,95672,95672,95672,Women's Shoes,Athletic,Athletic,Athletic,Athletic,Athletic,262144,N,-999,Clothing & Accessories,1251,1100,95672,9,1,Fashion
-1349,Collectibles:Decorative Collectibles:Decorative Collectible Brands:Avon:Bottles,0,,1,Other,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,156286,13779,1349,1349,1349,Decorative Collectibles,Decorative Collectible Brands,Avon,Bottles,Bottles,Bottles,262144,N,-999,Collectibles,-999,-999,1349,11,1,Collectibles
-1504,Business & Industrial:Electrical & Test Equipment:Test Equipment:Other,0,,9428,Other,4,Business (Office & Industrial),2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,12576,Business & Industrial,92074,4676,1504,1504,1504,1504,Electrical & Test Equipment,Test Equipment,Other,Other,Other,Other,262144,N,-999,Business (Office & Industrial),-999,-999,1504,4,1,Business & Industrial
-73506,Collectibles:Decorative Collectibles:Tea PotSets,0,,,,11,Collectibles,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,1,Collectibles,13777,73506,73506,73506,73506,73506,Decorative Collectibles,Tea PotSets,Tea PotSets,Tea PotSets,Tea PotSets,Tea PotSets,262144,N,-999,Collectibles,-999,-999,73506,11,1,Collectibles
-38238,Home & Garden:Home Decor & Accents:Wall Decor:Other,0,,,,36,Home & Living,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11700,Home & Garden,10033,10034,10034,10034,10034,10034,Home Decor,Other,Other,Other,Other,Other,262144,N,-999,Home Furnishing,2919,1868,10034,36,1,Home & Garden
-533,Coins & Paper Money:Coins: World:Africa:Other,0,,11116,Other,10,Coins,2008-06-03,2012-06-19 21:15:09,USER_X,USER_Y,11116,Coins & Paper Money,256,127144,533,533,533,533,Coins: World,Africa,Other,Other,Other,Other,262144,N,2431,Coins,6213,1620,533,10,1,Collectibles
-223,Toys & Hobbies:Diecast & Toy Vehicles:CarTrucks & Vans:Diecast-Modern Manufacture,0,,220,Other,28,Toys,2008-06-03,2013-04-28 20:37:19,USER_X,USER_Y,220,Toys & Hobbies,222,19036,223,223,223,223,Diecast & Toy Vehicles,CarTrucks & Vans,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,Diecast-Modern Manufacture,262144,N,-999,Toys,1740,1380,223,28,1,Collectibles
-31519,Computers/Tablets & Networking:Laptop & Desktop Accessories:Laptop Cases & Bags,3,,,,12,Computers,2012-05-01,2012-06-19 21:15:09,USER_X,USER_Y,58058,Computers/Tablets & Networking,31530,31519,31519,31519,31519,31519,Laptop & Desktop Accessories,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,Laptop Cases & Bags,262144,N,-999,Computers,6592,3189,31519,12,1,Electronics
-155226,ClothinShoes & Accessories:Women's Clothing:Sweats & Hoodies,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,155226,155226,155226,155226,155226,Women's Clothing,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,Sweats & Hoodies,262144,N,-999,Clothing & Accessories,-999,-999,155226,9,1,Fashion
-63889,ClothinShoes & Accessories:Women's Shoes:Mixed Items & Lots,0,,,,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,3034,63889,63889,63889,63889,63889,Women's Shoes,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,Mixed Items & Lots,262144,N,-999,Clothing & Accessories,-999,-999,63889,9,1,Fashion
-314,ClothinShoes & Accessories:Women's Clothing:Other,0,,11450,Other,9,Apparel,2008-06-03,2012-09-11 20:26:04,USER_X,USER_Y,11450,ClothinShoes & Accessories,15724,314,314,314,314,314,Women's Clothing,Other,Other,Other,Other,Other,262144,N,-999,Clothing & Accessories,-999,-999,314,9,1,Fashion
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv b/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
deleted file mode 100644
index 2544d8a..0000000
--- a/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
+++ /dev/null
@@ -1,402 +0,0 @@
-10000000157,2013-03-31,Auction,48027,0,12,184.21,1,10000001
-10000000158,2013-11-12,Others,164261,0,5,172.03,1,10000002
-10000000161,2013-04-06,Auction,82494,15,14,66.6,1,10000003
-10000000137,2013-05-17,Auction,66767,15,12,92.98,1,10000004
-10000000147,2013-05-20,FP-GTC,152801,0,5,132.33,1,10000005
-10000000155,2013-06-16,FP-GTC,43398,0,13,7.12,1,10000006
-10000000170,2013-06-14,Auction,95173,0,14,204.28,1,10000007
-10000000173,2013-03-22,Auction,158666,15,13,35.72,1,10000008
-10000000178,2013-03-10,Auction,12688,0,12,4.13,1,10000009
-10000000163,2013-11-01,FP-GTC,103324,15,5,27.48,1,10000010
-10000000166,2013-06-16,FP-GTC,108782,15,14,9.26,1,10000011
-10000000167,2013-09-12,Auction,80287,0,12,3.18,1,10000012
-10000000086,2013-09-28,Others,140746,100,13,3.18,1,10000013
-10000000110,2013-06-15,ABIN,87118,0,14,377.94,1,10000014
-10000000113,2013-03-14,Auction,25147,0,12,146.33,1,10000015
-10000000069,2013-09-01,FP-GTC,170302,15,5,51.23,1,10000016
-10000000079,2013-05-29,FP-non GTC,53064,0,13,72.65,1,10000017
-10000000080,2013-05-31,Auction,132939,0,13,66.6,1,10000018
-10000000130,2013-03-18,Auction,113593,15,12,9.26,1,10000019
-10000000268,2013-07-19,Auction,34273,100,14,583.44,1,10000020
-10000000132,2013-06-23,FP-GTC,106340,15,14,638.72,1,10000021
-10000000115,2013-05-20,FP-GTC,150265,15,14,4.54,1,10000022
-10000000117,2013-05-17,FP-GTC,24760,0,12,319.79,1,10000023
-10000000118,2013-03-11,Auction,37831,0,12,20.35,1,10000024
-10000000223,2013-01-30,FP-non GTC,1120,3,5,223.63,1,10000025
-10000000224,2013-01-26,FP-GTC,43972,100,13,204.28,1,10000026
-10000000243,2013-03-22,Auction,166013,15,14,5.48,1,10000027
-10000000217,2013-07-23,Auction,15568,15,14,27.48,1,10000028
-10000000218,2013-07-27,FP-GTC,103178,15,5,21.72,1,10000029
-10000000221,2013-10-29,ABIN,2023,0,12,3.18,1,10000030
-10000000256,2013-10-08,FP-GTC,94847,0,11,491.32,1,10000031
-10000000257,2013-04-26,Auction,15868,0,14,448.8,1,10000032
-10000000263,2013-01-01,Auction,32876,0,13,415.73,1,10000033
-10000000245,2013-01-15,Auction,62179,0,13,377.94,1,10000034
-10000000248,2013-05-27,FP-GTC,33038,15,14,146.33,1,10000035
-10000000254,2013-11-11,FP-GTC,156614,0,5,7.12,1,10000036
-10000000192,2013-03-08,Auction,106246,0,13,42.99,1,10000037
-10000000196,2013-03-25,Auction,20865,0,13,12.85,1,10000038
-10000000203,2013-08-20,FP-GTC,15115,0,13,55.89,1,10000039
-10000000179,2013-05-17,FP-GTC,3838,0,14,73.26,1,10000040
-10000000183,2013-06-05,Auction,759,0,11,112.56,1,10000041
-10000000185,2013-10-08,FP-non GTC,61323,0,11,3.49,1,10000042
-10000000211,2013-08-14,Auction,121153,0,13,184.21,1,10000043
-10000000213,2013-08-14,Auction,88750,0,13,157.14,1,10000044
-10000000214,2013-05-17,FP-GTC,161567,15,14,72.65,1,10000045
-10000000204,2013-08-09,FP-GTC,113802,15,14,51.23,1,10000046
-10000000208,2013-06-30,FP-non GTC,15808,15,14,15.85,1,10000047
-10000000209,2013-06-03,FP-GTC,174053,3,13,7.12,1,10000048
-10000000042,2013-12-31,Auction,2635,0,14,12.04,1,10000049
-10000000044,2013-12-25,Auction,1161,3,13,73.26,1,10000050
-10000000022,2013-03-28,FP-non GTC,64076,0,5,184.21,1,10000051
-10000000023,2013-01-30,FP-GTC,33977,15,13,172.03,1,10000052
-10000000047,2013-12-05,FP-GTC,31673,0,14,122.78,1,10000053
-10000000056,2013-10-08,Auction,174106,3,14,92.98,1,10000054
-10000000062,2013-12-27,Auction,26249,0,13,12.19,1,10000055
-10000000053,2013-12-16,FP-GTC,159184,0,5,15.65,1,10000056
-10000000055,2013-10-17,FP-GTC,10058,3,11,101.79,1,10000057
-10000000020,2013-11-17,ABIN,48904,0,12,7.12,1,10000058
-10000000007,2013-09-18,FP-non GTC,145970,0,14,12.85,1,10000059
-10000000008,2013-06-30,FP-GTC,963,0,13,12.19,1,10000060
-10000000002,2013-10-12,FP-GTC,118687,3,13,92.98,1,10000061
-10000000003,2013-08-20,FP-GTC,20886,0,14,42.99,1,10000062
-10000000010,2013-08-29,Auction,148324,15,13,1.88,1,10000063
-10000000016,2013-07-17,Auction,139255,15,14,21.14,1,10000064
-10000000017,2013-07-23,FP-GTC,20213,0,5,21.14,1,10000065
-10000000012,2013-01-06,Auction,32996,15,13,132.33,1,10000066
-10000000013,2013-08-14,FP-GTC,99985,0,14,120.87,1,10000067
-10000000067,2013-08-10,Auction,67703,3,14,120.87,1,10000068
-10000000085,2013-09-28,FP-non GTC,65,0,11,9.26,1,10000069
-10000000141,2013-08-21,FP-non GTC,130,0,14,16.26,1,10000070
-10000000078,2013-03-11,FP-GTC,164,0,14,157.14,1,10000071
-10000000109,2013-05-06,FP-GTC,216,0,11,1.88,1,10000072
-10000000096,2013-05-17,FP-non GTC,223,0,5,12.04,1,10000073
-10000000095,2013-01-10,FP-non GTC,223,0,14,189.23,1,10000074
-10000000098,2013-05-05,FP-non GTC,223,0,5,73.26,1,10000075
-10000000097,2013-02-03,FP-non GTC,223,0,5,4.13,1,10000076
-10000000099,2013-11-26,FP-non GTC,223,0,5,290.72,1,10000077
-10000000100,2013-08-30,FP-non GTC,223,0,5,265.56,1,10000078
-10000000126,2013-04-26,FP-GTC,279,15,5,5.91,1,10000079
-10000000252,2013-06-30,Auction,314,0,5,319.79,1,10000080
-10000000052,2013-06-30,Auction,314,211,5,246,1,10000081
-10000000253,2013-12-16,Auction,314,211,5,20.35,1,10000082
-10000000051,2013-12-15,Auction,314,0,5,36.7,1,10000083
-10000000190,2013-08-17,Auction,533,0,13,101.79,1,10000084
-10000000251,2013-12-15,ABIN,1349,0,5,47.71,1,10000085
-10000000050,2013-02-04,ABIN,1349,0,5,3.49,1,10000086
-10000000049,2013-01-11,ABIN,1349,0,13,46.44,1,10000087
-10000000250,2013-02-04,ABIN,1349,0,13,4.54,1,10000088
-10000000131,2013-05-17,ABIN,1357,0,14,3.18,1,10000089
-10000000172,2013-11-12,FP-GTC,1504,0,14,86.58,1,10000090
-10000000142,2013-08-21,FP-GTC,4943,0,13,12.85,1,10000091
-10000000195,2013-05-09,ABIN,6762,0,13,16.26,1,10000092
-10000000070,2013-09-19,Auction,9426,3,13,21.14,1,10000093
-10000000165,2013-02-06,FP-non GTC,10866,0,14,20.6,1,10000094
-10000000187,2013-02-02,Auction,11554,0,13,246,1,10000095
-10000000189,2013-08-23,FP-GTC,11848,0,14,109,1,10000096
-10000000139,2013-08-03,Auction,13836,0,13,39.41,1,10000097
-10000000140,2013-05-17,Auction,13836,0,14,16.26,1,10000098
-10000000102,2013-06-06,FP-GTC,13987,0,13,112.56,1,10000099
-10000000076,2013-07-02,Auction,15687,0,14,184.21,1,10000100
-10000000082,2013-10-25,Auction,15687,0,11,27.48,1,10000001
-10000000129,2013-04-20,FP-non GTC,16145,3,12,26.45,1,10000002
-10000000128,2013-03-12,FP-non GTC,16145,0,13,415.73,1,10000003
-10000000222,2013-03-28,ABIN,16509,0,5,56.36,1,10000004
-10000000021,2013-10-29,ABIN,16509,0,5,2.44,1,10000005
-10000000134,2013-05-22,FP-GTC,20485,0,14,269.76,1,10000006
-10000000135,2013-01-25,FP-GTC,20485,101,12,109,1,10000007
-10000000136,2013-06-12,FP-GTC,20485,101,12,101.79,1,10000008
-10000000241,2013-12-26,Auction,23446,23,14,246,1,10000009
-10000000041,2013-12-26,Auction,23446,23,14,189.23,1,10000010
-10000000242,2013-12-31,Auction,23446,23,14,15.65,1,10000011
-10000000040,2013-10-04,Auction,23446,23,14,28.23,1,10000012
-10000000194,2013-03-16,FP-GTC,24541,0,5,16.26,1,10000013
-10000000101,2013-05-21,FP-GTC,26262,0,5,122.78,1,10000014
-10000000077,2013-01-28,FP-GTC,30059,3,14,172.03,1,10000015
-10000000057,2013-04-26,Auction,31387,3,14,42.99,1,10000016
-10000000258,2013-10-06,Auction,31387,3,14,207.5,1,10000017
-10000000261,2013-11-06,FP-GTC,31519,0,14,5.91,1,10000018
-10000000058,2013-10-06,FP-GTC,31519,3,14,39.41,1,10000019
-10000000059,2013-12-28,FP-GTC,31519,0,14,16.26,1,10000020
-10000000060,2013-11-06,FP-GTC,31519,0,14,16.26,1,10000021
-10000000260,2013-11-06,FP-GTC,31519,0,14,78.48,1,10000022
-10000000259,2013-12-28,FP-GTC,31519,3,14,190.22,1,10000023
-10000000156,2013-06-11,FP-GTC,35570,100,12,2.44,1,10000024
-10000000119,2013-01-10,Auction,36250,0,5,7.12,1,10000025
-10000000186,2013-09-17,FP-non GTC,38238,0,14,36.7,1,10000026
-10000000038,2013-08-14,FP-GTC,40059,3,14,35.72,1,10000027
-10000000239,2013-08-09,FP-GTC,40059,3,14,3.49,1,10000028
-10000000034,2013-12-02,FP-GTC,41940,0,13,223.63,1,10000029
-10000000235,2013-02-01,FP-GTC,41940,0,13,265.56,1,10000030
-10000000127,2013-07-28,FP-non GTC,43479,0,13,62.02,1,10000031
-10000000103,2013-06-16,FP-GTC,44079,0,12,46.44,1,10000032
-10000000201,2013-08-23,Auction,45238,101,14,132.33,1,10000033
-10000000122,2013-06-15,Auction,45333,0,13,448.8,1,10000034
-10000000123,2013-06-15,FP-non GTC,45333,0,14,207.5,1,10000035
-10000000124,2013-06-01,FP-non GTC,45333,0,14,190.22,1,10000036
-10000000072,2013-08-10,FP-GTC,46575,0,14,16.71,1,10000037
-10000000043,2013-03-22,FP-non GTC,50508,0,13,4.13,1,10000038
-10000000244,2013-12-25,FP-non GTC,50508,0,13,1.88,1,10000039
-10000000121,2013-07-22,FP-GTC,50677,0,13,491.32,1,10000040
-10000000120,2013-04-13,FP-GTC,50677,0,5,2.44,1,10000041
-10000000168,2013-04-16,Auction,51582,0,14,56.36,1,10000042
-10000000073,2013-08-21,FP-GTC,57013,0,13,15.85,1,10000043
-10000000075,2013-04-22,FP-non GTC,57013,0,14,2.44,1,10000044
-10000000074,2013-08-29,FP-GTC,57013,0,14,7.12,1,10000045
-10000000093,2013-05-16,Auction,57784,0,14,35.72,1,10000046
-10000000265,2013-08-23,Auction,57990,3,11,9.26,1,10000047
-10000000266,2013-07-10,Auction,57990,3,14,3.18,1,10000048
-10000000267,2013-08-10,Auction,57990,3,14,638.72,1,10000049
-10000000065,2013-08-23,Auction,57990,3,14,141.7,1,10000050
-10000000143,2013-04-18,ABIN,57990,0,13,12.19,1,10000051
-10000000066,2013-07-10,Auction,57990,3,14,132.33,1,10000052
-10000000144,2013-06-16,ABIN,57990,3,5,5.48,1,10000053
-10000000064,2013-07-15,Auction,57990,3,11,1.88,1,10000054
-10000000061,2013-11-06,FP-GTC,60340,0,14,12.85,1,10000055
-10000000262,2013-12-27,FP-GTC,60340,0,14,62.02,1,10000056
-10000000019,2013-07-29,FP-GTC,60606,3,12,15.85,1,10000057
-10000000220,2013-11-17,FP-GTC,60606,3,12,9.26,1,10000058
-10000000018,2013-07-27,FP-GTC,60606,3,12,16.71,1,10000059
-10000000219,2013-07-29,FP-GTC,60606,3,12,20.6,1,10000060
-10000000145,2013-01-09,Auction,63861,3,5,1.88,1,10000061
-10000000200,2013-06-11,ABIN,63861,0,5,141.7,1,10000062
-10000000199,2013-01-10,ABIN,63861,0,5,1.88,1,10000063
-10000000237,2013-09-16,Others,63861,0,11,112.56,1,10000064
-10000000036,2013-01-14,Others,63861,0,11,94.45,1,10000065
-10000000125,2013-05-17,Auction,63861,0,14,78.48,1,10000066
-10000000198,2013-06-05,ABIN,63861,0,13,5.48,1,10000067
-10000000094,2013-05-24,Auction,63864,3,14,28.23,1,10000068
-10000000104,2013-05-15,Others,63889,0,13,3.49,1,10000069
-10000000107,2013-03-25,FP-GTC,67698,2,11,15.65,1,10000070
-10000000108,2013-03-09,FP-GTC,67698,0,11,5.48,1,10000071
-10000000106,2013-12-05,FP-GTC,67698,0,11,246,1,10000072
-10000000182,2013-04-18,FP-non GTC,73506,0,13,122.78,1,10000073
-10000000169,2013-11-01,FP-GTC,75665,0,14,223.63,1,10000074
-10000000146,2013-05-03,ABIN,75708,3,5,141.7,1,10000075
-10000000151,2013-04-21,FP-non GTC,80053,0,11,21.14,1,10000076
-10000000149,2013-03-12,FP-non GTC,80053,0,11,55.89,1,10000077
-10000000150,2013-05-19,FP-non GTC,80053,0,11,51.23,1,10000078
-10000000083,2013-11-23,Auction,80135,0,14,21.72,1,10000079
-10000000089,2013-10-19,Auction,95672,3,14,204.28,1,10000080
-10000000152,2013-05-18,Others,95672,0,11,21.14,1,10000081
-10000000035,2013-02-01,Others,100847,0,5,204.28,1,10000082
-10000000236,2013-01-14,Others,100847,0,5,122.78,1,10000083
-10000000090,2013-08-05,ABIN,139973,3,14,94.45,1,10000084
-10000000091,2013-05-19,ABIN,139973,0,11,86.58,1,10000085
-10000000033,2013-12-01,Auction,150047,3,14,56.36,1,10000086
-10000000234,2013-12-02,Auction,150047,3,14,290.72,1,10000087
-10000000249,2013-01-11,FP-GTC,155226,0,13,60.37,1,10000088
-10000000048,2013-05-27,FP-GTC,155226,0,13,112.56,1,10000089
-10000000181,2013-09-01,FP-GTC,156356,0,13,265.56,1,10000090
-10000000092,2013-04-11,FP-GTC,158798,0,11,35.72,1,10000091
-10000000191,2013-05-05,FP-non GTC,165888,0,13,92.98,1,10000092
-10000000229,2013-11-21,Auction,170083,3,11,28.23,1,10000093
-10000000028,2013-10-07,Auction,170083,3,11,27.48,1,10000094
-10000000031,2013-07-12,Auction,175750,3,14,9.26,1,10000095
-10000000032,2013-06-07,Auction,175750,3,14,3.18,1,10000096
-10000000177,2013-05-22,FP-GTC,175750,0,14,12.04,1,10000097
-10000000030,2013-11-28,Auction,175750,3,13,20.6,1,10000098
-10000000231,2013-07-12,Auction,175750,3,13,12.04,1,10000099
-10000000232,2013-06-07,Auction,175750,3,14,4.13,1,10000100
-10000000233,2013-12-01,Auction,175750,3,14,73.26,1,10000201
-10000000002,2012-10-12,Auction,48027,0,12,184.21,1,10000001
-10000000003,2012-08-20,Others,164261,0,5,172.03,1,10000002
-10000000007,2012-09-18,Auction,82494,15,14,66.6,1,10000003
-10000000008,2012-06-30,Auction,66767,15,12,92.98,1,10000004
-10000000010,2012-08-29,FP-GTC,152801,0,5,132.33,1,10000005
-10000000012,2012-01-06,FP-GTC,43398,0,13,7.12,1,10000006
-10000000013,2012-08-14,Auction,95173,0,14,204.28,1,10000007
-10000000016,2012-07-17,Auction,158666,15,13,35.72,1,10000008
-10000000017,2012-07-23,Auction,12688,0,12,4.13,1,10000009
-10000000018,2012-07-27,FP-GTC,103324,15,5,27.48,1,10000010
-10000000019,2012-07-29,FP-GTC,108782,15,14,9.26,1,10000011
-10000000020,2012-11-17,Auction,80287,0,12,3.18,1,10000012
-10000000021,2012-10-29,Others,140746,100,13,3.18,1,10000013
-10000000022,2012-03-28,ABIN,87118,0,14,377.94,1,10000014
-10000000023,2012-01-30,Auction,25147,0,12,146.33,1,10000015
-10000000028,2012-10-07,FP-GTC,170302,15,5,51.23,1,10000016
-10000000030,2012-11-28,FP-non GTC,53064,0,13,72.65,1,10000017
-10000000031,2012-07-12,Auction,132939,0,13,66.6,1,10000018
-10000000032,2012-06-07,Auction,113593,15,12,9.26,1,10000019
-10000000033,2012-12-01,Auction,34273,100,14,583.44,1,10000020
-10000000034,2012-12-02,FP-GTC,106340,15,14,638.72,1,10000021
-10000000035,2012-02-01,FP-GTC,150265,15,14,4.54,1,10000022
-10000000036,2012-01-14,FP-GTC,24760,0,12,319.79,1,10000023
-10000000038,2012-08-14,Auction,37831,0,12,20.35,1,10000024
-10000000040,2012-10-04,FP-non GTC,1120,3,5,223.63,1,10000025
-10000000041,2012-12-26,FP-GTC,43972,100,13,204.28,1,10000026
-10000000042,2012-12-31,Auction,166013,15,14,5.48,1,10000027
-10000000043,2012-03-22,Auction,15568,15,14,27.48,1,10000028
-10000000044,2012-12-25,FP-GTC,103178,15,5,21.72,1,10000029
-10000000047,2012-12-05,ABIN,2023,0,12,3.18,1,10000030
-10000000048,2012-05-27,FP-GTC,94847,0,11,491.32,1,10000031
-10000000049,2012-01-11,Auction,15868,0,14,448.8,1,10000032
-10000000050,2012-02-04,Auction,32876,0,13,415.73,1,10000033
-10000000051,2012-12-15,Auction,62179,0,13,377.94,1,10000034
-10000000052,2012-06-30,FP-GTC,33038,15,14,146.33,1,10000035
-10000000053,2012-12-16,FP-GTC,156614,0,5,7.12,1,10000036
-10000000055,2012-10-17,Auction,106246,0,13,42.99,1,10000037
-10000000056,2012-10-08,Auction,20865,0,13,12.85,1,10000038
-10000000057,2012-04-26,FP-GTC,15115,0,13,55.89,1,10000039
-10000000058,2012-10-06,FP-GTC,3838,0,14,73.26,1,10000040
-10000000059,2012-12-28,Auction,759,0,11,112.56,1,10000041
-10000000060,2012-11-06,FP-non GTC,61323,0,11,3.49,1,10000042
-10000000061,2012-11-06,Auction,121153,0,13,184.21,1,10000043
-10000000062,2012-12-27,Auction,88750,0,13,157.14,1,10000044
-10000000064,2012-07-15,FP-GTC,161567,15,14,72.65,1,10000045
-10000000065,2012-08-23,FP-GTC,113802,15,14,51.23,1,10000046
-10000000066,2012-07-10,FP-non GTC,15808,15,14,15.85,1,10000047
-10000000067,2012-08-10,FP-GTC,174053,3,13,7.12,1,10000048
-10000000069,2012-09-01,Auction,2635,0,14,12.04,1,10000049
-10000000070,2012-09-19,Auction,1161,3,13,73.26,1,10000050
-10000000072,2012-08-10,FP-non GTC,64076,0,5,184.21,1,10000051
-10000000073,2012-08-21,FP-GTC,33977,15,13,172.03,1,10000052
-10000000074,2012-08-29,FP-GTC,31673,0,14,122.78,1,10000053
-10000000075,2012-04-22,Auction,174106,3,14,92.98,1,10000054
-10000000076,2012-07-02,Auction,26249,0,13,12.19,1,10000055
-10000000077,2012-01-28,FP-GTC,159184,0,5,15.65,1,10000056
-10000000078,2012-03-11,FP-GTC,10058,3,11,101.79,1,10000057
-10000000079,2012-05-29,ABIN,48904,0,12,7.12,1,10000058
-10000000080,2012-05-31,FP-non GTC,145970,0,14,12.85,1,10000059
-10000000082,2012-10-25,FP-GTC,963,0,13,12.19,1,10000060
-10000000083,2012-11-23,FP-GTC,118687,3,13,92.98,1,10000061
-10000000085,2012-09-28,FP-GTC,20886,0,14,42.99,1,10000062
-10000000086,2012-09-28,Auction,148324,15,13,1.88,1,10000063
-10000000089,2012-10-19,Auction,139255,15,14,21.14,1,10000064
-10000000090,2012-08-05,FP-GTC,20213,0,5,21.14,1,10000065
-10000000091,2012-05-19,Auction,32996,15,13,132.33,1,10000066
-10000000092,2012-04-11,FP-GTC,99985,0,14,120.87,1,10000067
-10000000093,2012-05-16,Auction,67703,3,14,120.87,1,10000068
-10000000094,2012-05-24,FP-non GTC,65,0,11,9.26,1,10000069
-10000000095,2012-01-10,FP-non GTC,130,0,14,16.26,1,10000070
-10000000096,2012-05-17,FP-GTC,164,0,14,157.14,1,10000071
-10000000097,2012-02-03,FP-GTC,216,0,11,1.88,1,10000072
-10000000098,2012-05-05,FP-non GTC,223,0,5,12.04,1,10000073
-10000000099,2012-11-26,FP-non GTC,223,0,14,189.23,1,10000074
-10000000100,2012-08-30,FP-non GTC,223,0,5,73.26,1,10000075
-10000000101,2012-05-21,FP-non GTC,223,0,5,4.13,1,10000076
-10000000102,2012-06-06,FP-non GTC,223,0,5,290.72,1,10000077
-10000000103,2012-06-16,FP-non GTC,223,0,5,265.56,1,10000078
-10000000104,2012-05-15,FP-GTC,279,15,5,5.91,1,10000079
-10000000106,2012-12-05,Auction,314,0,5,319.79,1,10000080
-10000000107,2012-03-25,Auction,314,211,5,246,1,10000081
-10000000108,2012-03-09,Auction,314,211,5,20.35,1,10000082
-10000000109,2012-05-06,Auction,314,0,5,36.7,1,10000083
-10000000110,2012-06-15,Auction,533,0,13,101.79,1,10000084
-10000000113,2012-03-14,ABIN,1349,0,5,47.71,1,10000085
-10000000115,2012-05-20,ABIN,1349,0,5,3.49,1,10000086
-10000000117,2012-05-17,ABIN,1349,0,13,46.44,1,10000087
-10000000118,2012-03-11,ABIN,1349,0,13,4.54,1,10000088
-10000000119,2012-01-10,ABIN,1357,0,14,3.18,1,10000089
-10000000120,2012-04-13,FP-GTC,1504,0,14,86.58,1,10000090
-10000000121,2012-07-22,FP-GTC,4943,0,13,12.85,1,10000091
-10000000122,2012-06-15,ABIN,6762,0,13,16.26,1,10000092
-10000000123,2012-06-15,Auction,9426,3,13,21.14,1,10000093
-10000000124,2012-06-01,FP-non GTC,10866,0,14,20.6,1,10000094
-10000000125,2012-05-17,Auction,11554,0,13,246,1,10000095
-10000000126,2012-04-26,FP-GTC,11848,0,14,109,1,10000096
-10000000127,2012-07-28,Auction,13836,0,13,39.41,1,10000097
-10000000128,2012-03-12,Auction,13836,0,14,16.26,1,10000098
-10000000129,2012-04-20,FP-GTC,13987,0,13,112.56,1,10000099
-10000000130,2012-03-18,Auction,15687,0,14,184.21,1,10000100
-10000000131,2012-05-17,Auction,15687,0,11,27.48,1,10000001
-10000000132,2012-06-23,FP-non GTC,16145,3,12,26.45,1,10000002
-10000000134,2012-05-22,FP-non GTC,16145,0,13,415.73,1,10000003
-10000000135,2012-01-25,ABIN,16509,0,5,56.36,1,10000004
-10000000136,2012-06-12,ABIN,16509,0,5,2.44,1,10000005
-10000000137,2012-05-17,FP-GTC,20485,0,14,269.76,1,10000006
-10000000139,2012-08-03,FP-GTC,20485,101,12,109,1,10000007
-10000000140,2012-05-17,FP-GTC,20485,101,12,101.79,1,10000008
-10000000141,2012-08-21,Auction,23446,23,14,246,1,10000009
-10000000142,2012-08-21,Auction,23446,23,14,189.23,1,10000010
-10000000143,2012-04-18,Auction,23446,23,14,15.65,1,10000011
-10000000144,2012-06-16,Auction,23446,23,14,28.23,1,10000012
-10000000145,2012-01-09,FP-GTC,24541,0,5,16.26,1,10000013
-10000000146,2012-05-03,FP-GTC,26262,0,5,122.78,1,10000014
-10000000147,2012-05-20,FP-GTC,30059,3,14,172.03,1,10000015
-10000000149,2012-03-12,Auction,31387,3,14,42.99,1,10000016
-10000000150,2012-05-19,Auction,31387,3,14,207.5,1,10000017
-10000000151,2012-04-21,FP-GTC,31519,0,14,5.91,1,10000018
-10000000152,2012-05-18,FP-GTC,31519,3,14,39.41,1,10000019
-10000000155,2012-06-16,FP-GTC,31519,0,14,16.26,1,10000020
-10000000156,2012-06-11,FP-GTC,31519,0,14,16.26,1,10000021
-10000000157,2012-03-31,FP-GTC,31519,0,14,78.48,1,10000022
-10000000158,2012-11-12,FP-GTC,31519,3,14,190.22,1,10000023
-10000000161,2012-04-06,FP-GTC,35570,100,12,2.44,1,10000024
-10000000163,2012-11-01,Auction,36250,0,5,7.12,1,10000025
-10000000165,2012-02-06,FP-non GTC,38238,0,14,36.7,1,10000026
-10000000166,2012-06-16,FP-GTC,40059,3,14,35.72,1,10000027
-10000000167,2012-09-12,FP-GTC,40059,3,14,3.49,1,10000028
-10000000168,2012-04-16,FP-GTC,41940,0,13,223.63,1,10000029
-10000000169,2012-11-01,FP-GTC,41940,0,13,265.56,1,10000030
-10000000170,2012-06-14,FP-non GTC,43479,0,13,62.02,1,10000031
-10000000172,2012-11-12,FP-GTC,44079,0,12,46.44,1,10000032
-10000000173,2012-03-22,Auction,45238,101,14,132.33,1,10000033
-10000000177,2012-05-22,Auction,45333,0,13,448.8,1,10000034
-10000000178,2012-03-10,FP-non GTC,45333,0,14,207.5,1,10000035
-10000000179,2012-05-17,FP-non GTC,45333,0,14,190.22,1,10000036
-10000000181,2012-09-01,FP-GTC,46575,0,14,16.71,1,10000037
-10000000182,2012-04-18,FP-non GTC,50508,0,13,4.13,1,10000038
-10000000183,2012-06-05,FP-non GTC,50508,0,13,1.88,1,10000039
-10000000185,2012-10-08,FP-GTC,50677,0,13,491.32,1,10000040
-10000000186,2012-09-17,FP-GTC,50677,0,5,2.44,1,10000041
-10000000187,2012-02-02,Auction,51582,0,14,56.36,1,10000042
-10000000189,2012-08-23,FP-GTC,57013,0,13,15.85,1,10000043
-10000000190,2012-08-17,FP-non GTC,57013,0,14,2.44,1,10000044
-10000000191,2012-05-05,FP-GTC,57013,0,14,7.12,1,10000045
-10000000192,2012-03-08,Auction,57784,0,14,35.72,1,10000046
-10000000194,2012-03-16,Auction,57990,3,11,9.26,1,10000047
-10000000195,2012-05-09,Auction,57990,3,14,3.18,1,10000048
-10000000196,2012-03-25,Auction,57990,3,14,638.72,1,10000049
-10000000198,2012-06-05,Auction,57990,3,14,141.7,1,10000050
-10000000199,2012-01-10,ABIN,57990,0,13,12.19,1,10000051
-10000000200,2012-06-11,Auction,57990,3,14,132.33,1,10000052
-10000000201,2012-08-23,ABIN,57990,3,5,5.48,1,10000053
-10000000203,2012-08-20,Auction,57990,3,11,1.88,1,10000054
-10000000204,2012-08-09,FP-GTC,60340,0,14,12.85,1,10000055
-10000000208,2012-06-30,FP-GTC,60340,0,14,62.02,1,10000056
-10000000209,2012-06-03,FP-GTC,60606,3,12,15.85,1,10000057
-10000000211,2012-08-14,FP-GTC,60606,3,12,9.26,1,10000058
-10000000213,2012-08-14,FP-GTC,60606,3,12,16.71,1,10000059
-10000000214,2012-05-17,FP-GTC,60606,3,12,20.6,1,10000060
-10000000217,2012-07-23,Auction,63861,3,5,1.88,1,10000061
-10000000218,2012-07-27,ABIN,63861,0,5,141.7,1,10000062
-10000000219,2012-07-29,ABIN,63861,0,5,1.88,1,10000063
-10000000220,2012-11-17,Others,63861,0,11,112.56,1,10000064
-10000000221,2012-10-29,Others,63861,0,11,94.45,1,10000065
-10000000222,2012-03-28,Auction,63861,0,14,78.48,1,10000066
-10000000223,2012-01-30,ABIN,63861,0,13,5.48,1,10000067
-10000000224,2012-01-26,Auction,63864,3,14,28.23,1,10000068
-10000000229,2012-11-21,Others,63889,0,13,3.49,1,10000069
-10000000231,2012-07-12,FP-GTC,67698,2,11,15.65,1,10000070
-10000000232,2012-06-07,FP-GTC,67698,0,11,5.48,1,10000071
-10000000233,2012-12-01,FP-GTC,67698,0,11,246,1,10000072
-10000000234,2012-12-02,FP-non GTC,73506,0,13,122.78,1,10000073
-10000000235,2012-02-01,FP-GTC,75665,0,14,223.63,1,10000074
-10000000236,2012-01-14,ABIN,75708,3,5,141.7,1,10000075
-10000000237,2012-09-16,FP-non GTC,80053,0,11,21.14,1,10000076
-10000000239,2012-08-09,FP-non GTC,80053,0,11,55.89,1,10000077
-10000000241,2012-12-26,FP-non GTC,80053,0,11,51.23,1,10000078
-10000000242,2012-12-31,Auction,80135,0,14,21.72,1,10000079
-10000000243,2012-03-22,Auction,95672,3,14,204.28,1,10000080
-10000000244,2012-12-25,Others,95672,0,11,21.14,1,10000081
-10000000245,2012-01-15,Others,100847,0,5,204.28,1,10000082
-10000000248,2012-05-27,Others,100847,0,5,122.78,1,10000083
-10000000249,2012-01-11,ABIN,139973,3,14,94.45,1,10000084
-10000000250,2012-02-04,ABIN,139973,0,11,86.58,1,10000085
-10000000251,2012-12-15,Auction,150047,3,14,56.36,1,10000086
-10000000252,2012-06-30,Auction,150047,3,14,290.72,1,10000087
-10000000253,2012-12-16,FP-GTC,155226,0,13,60.37,1,10000088
-10000000254,2012-11-11,FP-GTC,155226,0,13,112.56,1,10000089
-10000000256,2012-10-08,FP-GTC,156356,0,13,265.56,1,10000090
-10000000257,2012-04-26,FP-GTC,158798,0,11,35.72,1,10000091
-10000000258,2012-10-06,FP-non GTC,165888,0,13,92.98,1,10000092
-10000000259,2012-12-28,Auction,170083,3,11,28.23,1,10000093
-10000000260,2012-11-06,Auction,170083,3,11,27.48,1,10000094
-10000000261,2012-11-06,Auction,175750,3,14,9.26,1,10000095
-10000000262,2012-12-27,Auction,175750,3,14,3.18,1,10000096
-10000000263,2012-01-01,FP-GTC,175750,0,14,12.04,1,10000097
-10000000265,2012-08-23,Auction,175750,3,13,20.6,1,10000098
-10000000266,2012-07-10,Auction,175750,3,13,12.04,1,10000099
-10000000267,2012-08-10,Auction,175750,3,14,4.13,1,10000100
-10000000268,2012-07-19,Auction,175750,3,14,73.26,1,10000201
[31/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/dw_category_grouping_names.dat
----------------------------------------------------------------------
diff --git a/dictionary/src/test/resources/dict/dw_category_grouping_names.dat b/dictionary/src/test/resources/dict/dw_category_grouping_names.dat
deleted file mode 100644
index 129c48d..0000000
--- a/dictionary/src/test/resources/dict/dw_category_grouping_names.dat
+++ /dev/null
@@ -1,448529 +0,0 @@
-Auto-Ersatz- & -Reparaturteile
-V?tements b?b?s, tout-petits
-Herrenschuhe
-Great Britain
-Fashion Jewelry
-Sachb?cher & Ratgeber
-Watches
-Postcards
-Herrenschuhe
-Women's Shoes
-Car, Truck Parts
-Rasenpflegeger?te
-T?l?, vid?o & audio domestique
-Uomo: Abbigliamento
-Damenmode
-Car Electronics
-Heavy Equipment
-Feste & Besondere Anl?sse
-Fashion Jewellery
-Bad & K?che
-Gardening
-Zeitschriften
-Kids' Clothing, Shoes & Accs
-Kids' Clothing, Shoes & Accs
-Film Photography
-V?tements pour hommes
-Kindermode, Schuhe & Access.
-Baby Clothing
-Tenue de danse
-Team Sports
-Auto-Ersatz- & -Reparaturteile
-Cycling
-Home Improvement
-Studium & Wissen
-Watches
-Womens' Clothing
-Tennis & sports de raquette
-MP3 Players
-Jewellery & Watches
-Makeup
-Kinderm?bel & Wohnen
-Digital Clocks & Clock Radios
-Loose Diamonds & Gemstones
-Damenmode
-Cleaning & Laundry Supplies
-Automobile
-Exonumia
-Handys ohne Vertrag
-Sciences, culture
-Weird Stuff
-Orologi da Polso: alla Moda
-K?rperpflege
-Dameskleding
-Cyclisme
-Movie Memorabilia
-CD's
-Standmodelle
-Lampes, plafonniers-?ventails
-Vinyl
-Reise & Regionales
-Games
-Animals
-Tv-componenten & Accessoires
-Pottery & China
-Paper
-????
-Damenmode
-Kleinger?te K?che
-Losse onderdelen PC
-DVD, Blu-ray & Heimkino
-Retro Home computers
-Men's Clothing
-Film DVD's
-Antiquarische B?cher
-Parfums
-Parts & Accessories
-TV & Home Audio Accessories
-TV- & Heim-Audio-Teile
-Produkcja i Przemys?
-Food & Beverages
-Other
-Digitalkameras
-Fragrances
-Nederlandse niet-euromunten
-Reise & Regionales
-Ricambi Auto
-Livres anciens - avant 1900
-Sports d'hiver
-Reit- & Fahrsport
-Baby Clothing
-Parfums
-Automobiles & camions
-Modelbouw
-Donna: Scarpe
-Gastro & Nahrungsmittelgewerbe
-Cz??ci samochodowe
-Cards
-Damenmode
-Printing & Personalization
-Vintage
-Veranstaltungs- & DJ-Equipment
-Kunststoffindustrie & Chemie
-Telefonia fissa
-Car Electronics
-Computer Components & Parts
-Fan Apparel & Souvenirs
-Drucker, Scanner & Zubeh?r
-Enterprise Networking, Servers
-Calzado mujer
-Damenmode
-Meubles, d?coration, Xx?me
-Bijoux de mode
-Handys ohne Vertrag
-R?seau d'entreprise, serveurs
-Playmobil
-Auto-Anbau- & -Zubeh?rteile
-Vehicle Electronics & GPS
-France
-Transport
-Hand Tools & Equipment
-Printing & Graphic Arts
-Winter Sports
-Transport
-Vinyles 33T
-Damenmode
-Redes de empresa y servidores
-Produktions- & Industriebedarf
-Damenmode
-Fotostudio-Zubeh?r
-Tools
-Accessoires de jeux vid?o
-????
-Wedding
-Jewellery Boxes & Supplies
-Composants PC
-Papier- en schrijfwaren
-Echtschmuck
-Nonfiction
-Europe
-Auto-Tuning & -Styling
-Imprimantes, scanneurs & acc.
-Petit Electro - Cuisine
-Vehicle Electronics & GPS
-Motorrad- & Kraftradteile
-Pottery
-TV, Video & Home Audio
-Auto-Hi-Fi & Navigation
-Damenmode
-Bolsos
-iPods & MP3 Players
-Cards
-Armband- & Taschenuhren
-Auto-Ersatz- & -Reparaturteile
-??/????
-Storage
-Sonstige
-Baugewerbe
-Metalware
-Handys ohne Vertrag
-Firmennetzwerk, Server
-Varie
-DVDs & Blu-rays
-Diecast & Toy Vehicles
-Motorrad- & Kraftradteile
-Damenschuhe
-Enterprise Networking, Servers
-Science Fiction & Horror
-Modellautos
-V?tements, accessoires
-Comics
-Historical Memorabilia
-Outdoor Toys
-Auto-Tuning & -Styling
-Metallbearbeitung & Schlosser.
-Drucker, Scanner & Zubeh?r
-Cars & Trucks
-Elektronik & Elektrotechnik
-Cars
-Film Photography
-Pi?ces & accessoires
-Militaria
-Vinyl en platen
-Gewerbe
-Engagement & Wedding
-Travaux de r?novation
-Dolls
-Computer-Komponenten & -Teile
-Accessories
-Automobilia
-Fashion Jewelry
-Armband- & Taschenuhren
-Festivals & Konzerte
-Miscellaneous
-Kleidung, Schuhe & Accessoires
-Coins
-Feste & Besondere Anl?sse
-Album e figurine
-Damen-Designerbekleidung
-Makeup
-Heavy Equipment
-Concerts
-R?seau d'entreprise, serveurs
-Baby Clothing
-Fashion Jewellery
-LCD / TFT Monitors
-Wereldkunst
-Sonstige
-Performance & DJ Equipment
-Laptop & Desktop Accessories
-Pi?ces & accessoires
-Herrenschuhe
-F?r Bastler
-Motorr?der
-Skin Care
-DVDs & Blu-rays
-Herrenmode
-M?bel
-Baby Clothing
-Reit- & Fahrsport
-Dekoration
-Echtschmuck
-Cars & Trucks
-Golf
-Am?rique latine
-Sports de plein air
-Parfums
-Automobilia
-DIY Materials
-Dolls
-Decorative Arts
-M?nzen Deutschland ab 1945
-Cleaning, Housekeeping
-Gitarren
-Shaving & Hair Removal
-Clothes, Shoes & Accessories
-Motorcycles
-Billets de spectacles
-LEGO
-CDs & DVDs
-Trading Cards
-Ringtones, Logos & Software
-Action Figures
-Kindermode, Schuhe & Access.
-Home Audio & HiFi Separates
-Haarpflege
-Kindermode, Schuhe & Access.
-Gesichtspflege
-Bijoux d'?poque, anciens
-Edelsteine
-Kindermode, Schuhe & Access.
-Jeux
-Clothes, Shoes & Accessories
-Clothes, Shoes & Accessories
-Litograf?as y Grabados
-Kleidung & Accessoires
-V?tements, accessoires
-Non-Fiction
-Meubles
-Voiture ancienne: pi?ces
-GSM's
-Fan Apparel & Souvenirs
-Kindermode, Schuhe & Access.
-Pro Audio
-??????/??
-Radsport
-Dolls
-Cards
-Hobby & K?nstlerbedarf
-Sachb?cher & Ratgeber
-Herenschoenen
-????
-DVDs & Blu-rays
-Reise & Regionales
-V?tements pour enfants
-Giochi Nintendo
-Outdoor Sports
-??
-Education & Professional
-Half Video Games
-Cookware, Dining & Bar
-Decorative Collectibles
-Fantaisie/mythe/magie
-Yard, Garden & Outdoor Living
-Non-Fiction
-Sonstige Branchen & Produkte
-Pro Audio Equipment
-Model Trains
-Fine Jewelry
-Enterprise Networking, Servers
-Fine Jewellery
-Decorative Collectibles
-Kitchenware
-Medizin & Labor
-Cross Stitch
-Familienplanung
-DVD, Blu-ray & Heimkino
-Baugewerbe
-Bi?uteria z pere?
-Histoire - philat?lie & poste
-Vehicle Electronics & GPS
-Video Game Accessories
-Yard, Garden & Outdoor Living
-Gastro & Nahrungsmittelgewerbe
-Bettausstattung
-Handys ohne Vertrag
-Cell Phone Accessories
-Motorcycles
-Mat?riel ?lectrique & d'essai
-Advertising
-Design & Stil
-Locaux: Mat?riel, Fournitures
-Tragbare Disc-Player & Radios
-Damenmode
-Cz??ci samochodowe: Tuning
-Kamine & Kaminzubeh?r
-Antiquarian & Collectible
-Kurzreisen
-??GSM??
-Motorrad- & Kraftradteile
-Parfums
-Cultural, Religious
-Clothes, Shoes & Accessories
-Radio Control
-Accessoires
-Electrical & Test Equipment
-Kleidung, Schuhe & Accessoires
-Damenmode
-Couture, Broderie, Tricot
-Mobile Accessories
-Kleidung, Schuhe & Accessoires
-Kleinger?te K?che
-Antiquarische B?cher
-Auto-Ersatz- & -Reparaturteile
-Baugewerbe
-Cars & Trucks
-??/????
-Herrenmode
-Gastro & Nahrungsmittelgewerbe
-Gro?handel & Sonderposten
-Arcade, Jukeboxes & Pinball
-Dameskleding
-Non-Fiction
-Llantas y Neum?ticos
-Motorr?der
-Furniture
-Breweriana
-Business & Management
-Damenmode
-Kindermode, Schuhe & Access.
-Cycling
-???
-Studium & Wissen
-CDs
-Dollhouse Miniatures
-Trading Cards
-Zeitschriften
-Laptop & Desktop Accessories
-Bijoux artisanaux
-Software
-Puppen
-Kindermode, Schuhe & Access.
-????????
-Test Auctions
-Photographie argentique
-Comics
-Autographs-Original
-Lotti e Stock Infanzia
-Damenmode
-Herrenmode
-Kindermode, Schuhe & Access.
-RC-Modellbau
-Kindermode, Schuhe & Access.
-Vintage & Antique Jewelry
-Albums BD
-Armband- & Taschenuhren
-Printers, Scanners & Supplies
-Technik & Photographica
-Oorlog
-Audio
-Men's Jewelry
-Pro Audio Equipment
-M?nzen Dt. Reich 1871-1945
-Kunststoffindustrie & Chemie
-Herrenmode
-Studium & Wissen
-Theatre Memorabilia
-Portable Audio & Headphones
-Poup?es
-Golf
-Digital Cameras
-K?nstlerbedarf
-Gadgets & Other Electronics
-Car Parts
-Odzie? m?ska
-Bettwaren, -w?sche & Matratzen
-Women's Clothing
-Heating/ Cooling/ Air
-Cards
-Postcards
-Postcards
-Mobiliar & Interieur
-Photographic Accessories
-V?tements b?b?s, tout-petits
-Skisport & Snowboarding
-Bistrot
-Cz??ci samochodowe
-Tools, Hardware & Locks
-Dekoration
-Kochen & Genie?en
-Handys mit Vertrag
-Digitalkameras
-P?yty kompaktowe
-Fantasy, Mythical & Magic
-T?l?, vid?o & audio domestique
-Gro?handel & Sonderposten
-Greeting Cards & Party Supply
-Gro?handel & Sonderposten
-Pi?ces & accessoires
-Fan Apparel & Souvenirs
-Handys mit Vertrag
-TV- & Heim-Audio-Zubeh?r
-Nintendo overig
-Handy- & PDA-Zubeh?r
-Modellbaus?tze
-Nonfiction Books
-Reise & Regionales
-Drucker, Scanner & Zubeh?r
-Computer Components & Parts
-Miscellaneous
-Transportation
-Fan Shop
-Sewing/ Fabric/ Textiles
-Restaurant & Catering
-Agrar, Forst & Kommune
-Diecast & Toy Vehicles
-Computer Components & Parts
-Damenmode
-Th?mes
-V?tements & souvenirs de fan
-Haushalt
-Radsport
-Science-Fiction
-Antiquarische B?cher
-Drives, Storage & Blank Media
-Rollenspiele & Tabletops
-V?tements, accessoires
-Collections
-Haushalt
-Car Electronics
-Automobilia
-Odzie? pozosta?a
-Sauna & Schwimmbad
-Beeldende kunst
-Makeup
-Kleinger?te K?che
-Guitar
-Saggistica
-Electrical & Test Equipment
-Manik?re & Pedik?re
-Damenmode
-Needlecrafts & Yarn
-Komunikacja i ??czno??
-Kleidung, Schuhe & Accessoires
-Fan Apparel & Souvenirs
-Echtschmuck
-Back?fen & Herde
-Dolls & Toys
-Baugewerbe
-Autographs
-Weitere Ballsportarten
-Home Audio & HiFi Separates
-Massage
-Bedding
-Motorcycles
-BD
-Tennis & Racquet Sports
-Women's Clothing
-Football Memorabilia
-Reptilien
-????
-????DIY
-Elektronika i elektrotechnika
-Parfums
-B?rom?bel
-Paper
-Stamps
-Timeshare
-Kids' Clothes, Shoes & Accs.
-Cz??ci motocyklowe
-Skisport & Snowboarding
-Decorative Arts
-USB
-Boats
-Z?une & Sichtschutzw?nde
-Pianos, Keyboards & Organs
-Kindermode, Schuhe & Access.
-????
-Livres de bandes dessin?es
-V?tements pour enfants
-Bew?ssern
-Games
-Porzellan & Keramik
-Gefrierger?te & K?hlschr?nke
-Gartenger?te
-Clothes, Shoes & Accessories
-Bijoux de mode
-Fan Shop
-Dolls
-Fernseher
-Damenschuhe
-Manuals & Guides
-Computer-Komponenten & -Teile
-Figuren
-Games
-Comics
-Kids' Clothing, Shoes & Accs
-Women's Shoes
-PC- & Videospiele
-Trains/ Railway Models
-Hard Disk Drives
-Fitness
-Rugs & Carpets
-Memorabilia
-B?rotechnik
-Damenmode
-Stationery
-Animation Art, Characters
-Relojes de Pulsera
-Golf
-Accessori laptop e desktop
-DVDs & Blu-ray Discs
-S?mereien & Zwiebeln
-Kleinger?te K?che
-Damenmode
-Porzellan & Keramik
-Weiterer Wassersport
-DVDs & Blu-ray Discs
-MRO & Industrial Supply
-Batteries
-Armband- & Taschenuhren
-Arts, Artisanat, D?coration
-Damenmode
-Gitarren
-DVDs & Blu-rays
-Water Sports
-Test Auctions
-Kleidung, Schuhe & Accessoires
-Cards
-Guitar
-Radsport
-Fashion Jewellery
-Femmes: Chaussures
-Fine Jewellery
-Men's Clothing
-Trading Card Games
-Hobby & K?nstlerbedarf
-Comic Books
-Football Shirts
-P?riodiques, Magazines BD
-Audio- & Video-Rarit?ten
-Film Photography
-Automobile
-Baugewerbe
-Kindermode, Schuhe & Access.
-Chaussures pour hommes
-Kleidung, Schuhe & Accessoires
-Computer-Komponenten & -Teile
-Holiday & Seasonal
-Office
-Motorrad- & Kraftradteile
-Heim-Audio & HiFi
-Echtschmuck
-Soins du Corps & Hygi?ne
-Animals
-Networking e reti home
-Vintage Sports Memorabilia
-Personnages de t?l? & de films
-Pottery
-Trading Cards
-Serien & Lizenzprodukte
-Motorcycle Parts & Accessories
-Logiciels
-Services & Real Estate
-Miniatures
-T?l?phones ? domicile
-Casino
-LCD TVs
-Weitere Ballsportarten
-Muziek luisteren
-Guitare
-Cartes de collection
-Sonnenschutz
-Kleinger?te K?che
-Antiquarische B?cher
-Teich- & Bachlaufsysteme
-Auto-Tuning & -Styling
-Baby Feeding
-TV, Video & Home Audio
-Auto-Anbau- & -Zubeh?rteile
-Mariage & tenue habill?e
-Herrenschmuck
-Silver
-Damenmode
-Fine Jewellery
-Men's Accessories
-Damenmode
-Kinderkleding- meisjes
-Teich- & Bachlaufsysteme
-Kitchen, Dining & Bar
-Sports motoris?s
-Vins
-Zabawki
-Porcelain/ China
-Laptops & Netbooks
-Anlageobjekte/Versteigerungen
-Damenmode
-Auto-Ersatz- & -Reparaturteile
-Souvenirs ?missions t?l?vis?es
-Militaria
-Overige games & Consoles
-Armband- & Taschenuhren
-Bijoux pour hommes
-Moto e Scooter
-Reading Accessories
-Car, Truck Parts
-TV, Movie & Character Toys
-Damenmode
-Auto-Ersatz- & -Reparaturteile
-Lighting & Studio
-Geb?udebaus?tze
-Camera & Photo Accessories
-Cardmaking & Scrapbooking
-Modernariato
-Model Trains
-Film DVD's
-Accessori cellulari e palmari
-Car Parts
-Plants/ Seeds/ Bulbs
-Vintage-Mode
-Kleidung, Schuhe & Accessoires
-Telescopios y prism?ticos
-Records
-Kleidung, Schuhe & Accessoires
-Brass
-Pauschalreisen
-RC-Modellbau
-Fashion Jewellery
-Budownictwo i akcesoria
-???
-Kindermode, Schuhe & Access.
-Women's Clothing
-Objets de collection de d?cora
-Playstation 2
-Gastro & Nahrungsmittelgewerbe
-Building Materials, DIY
-Nursery Decor
-Pi?ces & accessoires
-Jeux d'int?rieur
-Auto-Ersatz- & -Reparaturteile
-Cz??ci samochodowe: Tuning
-Transportation
-Other Jewellery
-Koffer, Taschen & Accessoires
-Diecast & Toy Vehicles
-Revues, manuels, catalogues
-Audio portatifs & ?couteurs
-Transport
-Fish & Aquarium
-Bambini 2 - 16 anni
-Exercise & Fitness
-Music Memorabilia
-Barbie
-Hair Care & Styling
-Laptop & Desktop Accessories
-Printers, Scanners & Supplies
-Autres Sports
-Auto-Tuning & -Styling
-?iEI
-Kunst & Kultur
-M?nzen Europa
-Clothing: Girls Jr
-Antikspielzeug
-Teleskope & Ferngl?ser
-?????????
-Bottles & Insulators
-Werkzeuge & Werkstattbedarf
-????
-Engagement & Wedding
-Skisport & Snowboarding
-Kindermode, Schuhe & Access.
-Kids' Clothes, Shoes & Accs.
-Zubeh?r
-Damenmode
-Auto-Ersatz- & -Reparaturteile
-K?rperpflege
-Eisenwaren
-Pflanzen, B?ume & Str?ucher
-Men's Clothing
-Computer-Komponenten & -Teile
-iPods & MP3-Player
-Modellautos
-M?bler
-Motorradkleidung
-Rennbahnen & Slotcars
-Sachb?cher & Ratgeber
-Asian Art
-Loose Diamonds & Gemstones
-Damenmode
-Kost?me & Verkleidungen
-Werkzeug
-Skisport & Snowboarding
-Car, Truck Parts
-Imprimantes, scanneurs & acc.
-Cookware, Dining & Bar
-Gadgets & autres ?lectroniques
-Cucina
-Disneyana
-Education & Professional
-Weitere Sportarten
-Componenti e parti
-Vintage
-Camping & Outdoor
-Kindermode, Schuhe & Access.
-Historische Wertpapiere
-Furniture
-Technik & Ger?te
-Vintage
-Kabel & Steckverbinder
-Women's Clothing
-Scolaire, dictionnaires
-Parts & Accessories
-Records
-Historical Memorabilia
-CD
-Makeup
-Motorr?der
-Cards
-Kinderwagen & -tragen
-Figurines d'action
-Kleidung, Schuhe & Accessoires
-Kunst & Kultur
-Cuisine, restaurants & bar
-Manik?re & Pedik?re
-Test Auctions
-Pet Accessories
-Handy- & PDA-Zubeh?r
-Fu?ball-Fanshop
-Guitare
-Zahnpflege
-Holzspielzeug
-Damenmode
-Outdoor Sports
-Auto-Anbau- & -Zubeh?rteile
-Guitar
-Haushalt & K?che
-iPods & MP3 Players
-Historical Memorabilia
-Transportation
-Construction Toys & Kits
-Boats
-Postcards
-Herrenschuhe
-Children's & Young Adults
-Advertising
-Auto-Ersatz- & -Reparaturteile
-Verre, cristal
-Fu?ball
-Golf
-Business & Management
-Bath
-Motorrad- & Kraftradteile
-Manufacturiers & m?tallurgie
-Onderdelen Personenauto's
-Men's Jewelry
-Auto-Ersatz- & -Reparaturteile
-Filmposter
-DVDs & Blu-rays
-Cell Phone Accessories
-Echtschmuck
-TV-Receiver & Set-Top-Boxen
-Jewellery Design & Repair
-Desktops & All-In-Ones
-Football Programmes
-Filters
-Girls' Clothing
-Nederland
-Motorrad- & Kraftradteile
-Software
-Fishing
-Speakers, Webcams & Multimedia
-Haarpflege
-Pro Audio Equipment
-Damenmode
-Womens' Handbags & Bags
-Auto-Tuning & -Styling
-Drucker, Scanner & Zubeh?r
-Edelsteine
-Motorrad- & Kraftradteile
-Software
-Gesichtspflege
-Men's Shoes
-Laufwerke & Speichermedien
-Nachschlagewerke
-Books & Manuscripts
-Skisport & Snowboarding
-Yard, Garden & Outdoor Living
-Horse Riding
-Kochen & Genie?en
-Men's Accessories
-CDs
-Skisport & Snowboarding
-Ansichtskarten
-Wholesale Lots
-Calendriers, tickets, affiches
-Fish & Aquariums
-Kids' Clothes, Shoes & Accs.
-VHS-Kassetten
-Kids' Clothing, Shoes & Accs
-Golf
-Transport
-Auto-Ersatz- & -Reparaturteile
-United States
-Fleisch & Wurst
-Yard, Garden & Outdoor Living
-Computer-Komponenten & -Teile
-Auto-Anbau- & -Zubeh?rteile
-Golf
-CD
-Animation Art & Characters
-Kids' Clothes, Shoes & Accs.
-V?tements, accessoires
-Non-Fiction
-H?rb?cher & H?rspiele
-Fan Apparel & Souvenirs
-Construction
-T?l?, vid?o & audio domestique
-Damenmode
-Modelleisenbahn
-KFZ-Handwerk
-Antiquarische B?cher
-Enthaarung & Rasur
-Kindermode, Schuhe & Access.
-Test Auctions
-Geheugenkaarten
-R?seaux
-Hi-Fi, son, mat?riel audio
-Monedas Antiguas
-Bettausstattung
-Coins
-Transportation
-Testbereich
-Damenmode
-Damenmode
-Computer-Komponenten & -Teile
-Luxus-Accessoires
-Handys ohne Vertrag
-Non-Fiction
-Computer-Komponenten & -Teile
-Medizin & Labor
-Car, Truck Parts
-Dekoration
-Fine Jewellery
-Kochen & Genie?en
-Trading Card Games
-Men's Accessories
-Cards
-Transportation
-Postcards
-DIY Materials
-Memorabilia
-Chasse
-Jantes & Pneus
-Componenti e parti
-PC- & Videospiele
-Handys mit Vertrag
-Kindermode, Schuhe & Access.
-Ladenausstattung & -werbung
-Digitalkameras
-Logiciels
-Hobby & K?nstlerbedarf
-Sachb?cher & Ratgeber
-Funktechnik
-Gardening
-Jongenskleding
-Non-Fiction Books
-Pi?ces Etrang?res
-Fashion Jewellery
-Home Networking & Connectivity
-Football Memorabilia
-Input Peripherals
-Kindermode, Schuhe & Access.
-Commonwealth/ British Colonial
-Reit- & Fahrsport
-Kitchen, Dining & Bar
-Digital Cameras
-Modelleisenbahn
-Kindermode, Schuhe & Access.
-Dekoration
-??/??
-Diecast
-Lenses & Filters
-TV, Video & Home Audio
-Accessoires pour femmes
-Bad & K?che
-Heavy Equipment Attachments
-Nachschlagewerke
-Fashion Jewelry
-Beleuchtung
-Swimming
-Ansichtskarten
-DVD's
-Food & Beverages
-Bureau
-TV & Film Character Toys
-Business & Management
-Beleuchtung
-Automobilia
-Historische Wertpapiere
-Fan Apparel & Souvenirs
-Women's Clothing
-Heim-Audio & HiFi
-Feste & Besondere Anl?sse
-Test Auctions
-Souvenirs de jeu usag?s
-Malerei
-?berraschungseier
-Oprogramowanie
-Gesichtspflege
-Pro-Audio Equipment
-Budownictwo
-Kleidung, Schuhe & Accessoires
-Sports de plein air
-Echtschmuck
-Motorcycle Parts & Accessories
-Maison & jardin
-Hunting
-Other Formats
-Powersports
-Baby Clothing
-Claviers, souris, pointeurs
-Piel?gnacja jamy ustnej
-Wholesale Lots
-B?rotechnik
-Historische Wertpapiere
-Elektronik & Elektrotechnik
-Software
-Records
-Fan Apparel & Souvenirs
-Antiques 20th Century
-Design & Stil
-Exotische Instrumente
-MRO & Industrial Supply
-Auto's
-Cz??ci samochodowe
-Sports d'?quipe
-Odzie? damska
-Autogramme & Autographen
-Radio Control & Control Line
-Fan Apparel & Souvenirs
-P?yty kompaktowe
-Heim-Audio & HiFi
-Car Tuning & Styling
-Kindermode, Schuhe & Access.
-%F%l%S%2!<%`
-Multipurpose Batteries & Power
-Reise & Regionales
-Advertising
-Antique Furniture
-Music Memorabilia
-Shaving & Hair Removal
-Parts & Accessories
-Vintage Clothing & Accessories
-Comics
-Manuscripts
-Motorrad- & Kraftradteile
-Laufwerke & Speichermedien
-Herrenschuhe
-Accessoires t?l. mobiles, PDA
-Radsport
-Camping
-Porzellan & Keramik
-Yard, Garden & Outdoor Living
-Cell Phone Accessories
-Tatouages & Art corporel
-Furniture
-Men's Clothing
-Boy's Accessories
-Antique Furniture
-Antiquities
-Animals
-Portable Audio & Headphones
-Armband- & Taschenuhren
-Handys ohne Vertrag
-Obr?bka drewna i stolarstwo
-Heim-Audio & HiFi
-Portables, netbooks
-Automobile
-Handys ohne Vertrag
-Kindermode, Schuhe & Access.
-Reise & Regionales
-Golf
-Belletristik
-Religion & Spirituality
-Laufwerke & Speichermedien
-Pi?ces Euro
-Men's Clothing
-Antique Furniture
-V?hicules miniatures
-Industrial Tools
-Outdoor Sports
-Fitness
-Zubeh?r
-Elektronik & Elektrotechnik
-Holiday & Seasonal
-Accessories
-Yard, Garden & Outdoor Living
-Fu?ball
-Animals
-Aliment & boissons
-Women's Shoes
-Literatur & Kochb?cher
-Automobilia
-Vinyl
-Tools
-Action Figures
-Gesichtspflege
-Vorsorge & Entspannung
-????
-Hunde
-TV- & Heim-Audio-Zubeh?r
-Bain & soins du corps
-Computer Components & Parts
-Europe
-Coins
-T?l?, vid?o & audio domestique
-Fragrances
-Manuals & Resources
-Produktions- & Industriebedarf
-Audio- & Video-Rarit?ten
-Motorcycle Parts & Accessories
-Books
-Sports Cars
-Europe
-Preschool Toys & Pretend Play
-Non Fiction
-Computer-Komponenten & -Teile
-????
-Animals
-Reit- & Fahrsport
-Korsord & korsordsrelaterat
-Studium & Wissen
-Cat?gorie test
-Cars & Trucks
-Women's Shoes
-M?nzen Dt. Reich 1871-1945
-Damenmode
-Computer Components & Parts
-Fan Shop
-Vintage
-Children's Books
-Veranstaltungs- & DJ-Equipment
-Autoreifen & Felgen
-Motorrad- & Kraftradteile
-Damenmode
-Computer-Komponenten & -Teile
-Kinderkleding- meisjes
-Outdoor Sports
-Standmodelle
-Magazines
-Unisex Jewellery
-Klein- & H?ngeaufbewahrung
-Guitar
-Reit- & Fahrsport
-Kleinger?te K?che
-????
-Cz??ci samochodowe
-Kunststoffindustrie & Chemie
-Damenmode
-Printers & Scanners
-Zubeh?r & Aufbewahrung
-Reise & Regionales
-Auto-Anbau- & -Zubeh?rteile
-Medizin & Labor
-V?tements/accessoires d'?poque
-Herrenschuhe
-Kochen & Genie?en
-Herrenschuhe
-M?bel
-Damenmode
-Poligrafia
-L?sungsb?cher & Ratgeber
-D?coration de maison
-Holzbearbeitung & Tischlerei
-Back?fen & Herde
-Herrenschmuck
-Skin Care
-Fu?ball
-Coins: World
-Action- & Spielfiguren
-Engagement & Wedding
-Home Improvement
-Books/Music/Kids Book
-Damenmode
-Games
-Greeting Cards & Party Supply
-Animals
-Reit- & Fahrsport
-Pauschalreisen
-DVDs & Blu-rays
-Make-up
-Kleidung, Schuhe & Accessoires
-Accessoires de voyage
-Body Jewellery
-Animals
-Parts & Accessories
-Sachb?cher & Ratgeber
-Software
-Damen-Designerbekleidung
-Lodging
-Automobilia
-Magazines
-CDs
-Home Networking & Connectivity
-Vintage & Antique Jewelry
-Laptops
-Trading Cards/ CCG
-Armband- & Taschenuhren
-Damenmode
-VHS Tapes
-Kitchen, Dining & Bar
-Kindermode, Schuhe & Access.
-Herrenschuhe
-Education & Professional
-DVDs & Blu-rays
-Makeup
-Fan Apparel & Souvenirs
-Baby & Toddler Clothing
-Heimnetzwerk & Zubeh?r
-Packing & Shipping
-Bad & K?che
-Damenmode
-Piercings
-B?rotechnik
-CDs
-Bears
-??DVD
-Ladenausstattung & -werbung
-Kindermode, Schuhe & Access.
-Packing & Shipping
-PC- & Videospiele
-Motorr?der
-M?bel
-??/????
-Gsm-accessoires & Onderdelen
-Handys mit Vertrag
-Religious
-Fisch & Meeresfr?chte
-Loisirs Vie Pratique
-Bricolage
-Soins de sant?, sciences vie
-Damenmode
-Baby Clothing
-Werkzeuge & Werkstattbedarf
-Multipurpose Batteries & Power
-Malerei
-Objektive & Filter
-Cards
-Grues: logistiq., manutention
-Wedding
-Cards
-Women's Clothing
-Bad & K?che
-Holiday & Seasonal
-Handys ohne Vertrag
-Cycling
-Deutschland
-Dekoration
-Steiff
-Inline Skates
-CDs
-Auto-Ersatz- & -Reparaturteile
-Clothing: Men
-Parts & Accessories
-Accessoires t?l. mobiles, PDA
-Women's Clothing
-Bettwaren, -w?sche & Matratzen
-Piercing-/K?rperschmuck
-Weitere Ballsportarten
-Books & Manuscripts
-Model Kits
-TV-Receiver & Set-Top-Boxen
-Antikspielzeug
-Handys ohne Vertrag
-Camera & Photo Accessories
-Packing & Shipping
-Vinyles 33T
-Indian Coins
-Heavy Equipment
-Installation
-Sammleruhren
-Music Memorabilia
-V?tements & souvenirs de fan
-Studium & Wissen
-Cz??ci motocyklowe
-???
-Accessories
-Analoge Fotografie
-Unisex kleding
-Action Figures
-Camping & Outdoor
-Wedding Supplies
-Dekoration
-Video Games
-Outdoor Sports
-DVDs & Blu-rays
-Services & Real Estate
-Komunikacja i ??czno??
-Clothing & Accessories
-?????
-Outdoor Toys & Structures
-Edelsteine
-Asiatische Lebensmittel
-Herrenschuhe
-Cyclisme
-Advertising
-Objets de collection sports
-Clothing & Accessories
-Furniture
-Damenmode
-Classic Cars
-Dolls
-Echtschmuck
-Damenmode
-Bathroom
-Fournitures pour chien
-Handy- & PDA-Zubeh?r
-Antiquarische B?cher
-Restaurant & Catering
-Veranstaltungs- & DJ-Equipment
-Speakers, Webcams & Multimedia
-CD
-Damenmode
-Other Tickets, Travel
-Antiquarische B?cher
-Make Up e Cosmetici
-Logiciels
-TV- & Heim-Audio-Zubeh?r
-Motive
-Modeschmuck
-Skisport & Snowboarding
-Fan Apparel & Souvenirs
-Vins Alcools
-Men's Clothing
-Objektive & Filter
-Schule & Ausbildung
-Vintage Computing
-D?coration de maison
-Games
-Makeup
-Nonfiction
-Antiquarische B?cher
-Clothes, Shoes & Accessories
-Dolls' House Miniatures
-Video Game Accessories
-Jet-Ski, Acquascooter
-Drives, Storage & Blank Media
-Car Accessories & Parts
-Skisport & Snowboarding
-Cards
-Playmobil
-Fashion Jewelry
-Motocyclettes
-Restaurant & Catering
-Transportation
-Reklame & Werbung
-Nintendo DS
-Jewellery
-Radsport
-????
-Motorcycle Parts & Accessories
-Exercise & Fitness
-Handy- & PDA-Zubeh?r
-Urz. wskazuj?ce i wej?ciowe
-Fashion Jewelry
-Diecast & Vehicles
-Software
-Dancewear & Accessories
-Instruments d'?criture, plumes
-Sachb?cher & Ratgeber
-Vintage & Antique Jewelry
-Coins: US
-Sony
-CDs
-Mat?riel l?ger & outils
-Sewing (1930-Now)
-????
-Tasteninstrumente
-Kleinger?te Haushalt
-Electrical & Test Equipment
-B?rom?bel
-Studium & Wissen
-BD
-Greeting Cards & Party Supply
-Herrenschmuck
-Software
-??????
-Computer-Komponenten & -Teile
-Weitere Ballsportarten
-??
-Medizin & Labor
-Other
-Kids' Clothes, Shoes & Accs.
-Shaving & Hair Removal
-Baustoffe
-Art du XIX?me, et avant
-Pi?ces & accessoires
-Laufwerke & Speichermedien
-Damenmode
-iPods y reproductores MP3
-TV Memorabilia
-Motorrad- & Kraftradteile
-Rollenspiele & Tabletops
-????
-Veranstaltungs- & DJ-Equipment
-Software
-Fashion Jewellery
-Cardmaking & Scrapbooking
-Elektronik & Elektrotechnik
-Radio Communication
-????
-Fu?ball-Fanshop
-Kleidung, Schuhe & Accessoires
-Bielizna damska,st. k?pielowe
-Handys ohne Vertrag
-Loose Diamonds & Gemstones
-Cars
-Objets Collectors
-Photographie argentique
-Men's Accessories
-V?hicules miniatures
-Radsport
-Coins: US
-Fashion Jewelry
-Spielzeug f?r drau?en
-Poterie & porcelaine
-Men's Shoes
-Elektronik & Elektrotechnik
-Damen-Accessoires
-Non-Fiction
-MRO & Industrial Supply
-Kids' Clothing, Shoes & Accs
-Budownictwo
-Home Networking & Connectivity
-Ansichtkaarten en foto's
-Building Materials & Supplies
-MP3 Player Accessories
-Outdoor Sports
-Linge & textile (avant 1930)
-Cars & Trucks
-Papier, B?ro- & Schreibwaren
-Kinder- & Jugendliteratur
-Mobile Phone Accessories
-Bettwaren, -w?sche & Matratzen
-Gesichtspflege
-GPS & In-Car Technology
-Beads & Jewellery Making
-Transport
-Golf
-Playmobil
-Armband- & Taschenuhren
-Damenmode
-Fan Apparel & Souvenirs
-Fancy Dress & Period Costume
-Tools
-Trading Card Games
-T?l?phones mobiles
-Akcesoria dla artyst?w
-Sonstige Branchen & Produkte
-Teppiche & Teppichb?den
-Edelsteine
-Electrical & Test Equipment
-Tattoos & Body Art
-Sports de plein air
-Computer-Komponenten & -Teile
-Kindermode, Schuhe & Access.
-DVD - Films
-TV- & Heim-Audio-Zubeh?r
-Modelarstwo
-Echtschmuck
-Hobby & K?nstlerbedarf
-Rugs & Carpets
-Records
-Watches
-Armband- & Taschenuhren
-Car, Truck Parts
-Haarpflege
-Kleidung, Schuhe & Accessoires
-Handys ohne Vertrag
-???????
-Team Sports
-Cardmaking & Scrapbooking
-Damentaschen
-Fashion Jewellery
-Varie
-Echtschmuck
-Auto-Tuning & -Styling
-Kitchen, Dining, Bar
-Tapis & moquettes
-Music Memorabilia
-Ricambi Auto
-Handy- & PDA-Zubeh?r
-Computer Components & Parts
-Kleidung, Schuhe & Accessoires
-Comics
-Needlecrafts & Yarn
-Composants & pi?ces d'ordi
-Soins de sant?, sciences vie
-V?tements & souvenirs de fan
-Damenmode
-Kamera- & Fotozubeh?r
-??????/??
-Cycling
-Makeup
-Drucker, Scanner & Zubeh?r
-Ansichtskarten
-Non-Fiction
-Gewerbe
-Lighting & Studio
-Non-Fiction
-Sport
-Pauschalreisen
-Complementos Mujer
-Scrapbooking & Paper Crafts
-Revues, manuels, catalogues
-Mobiliar & Interieur
-Blitzger?te & Zubeh?r
-Kindermode, Schuhe & Access.
-Genealogy
-Militaria
-Kurzreisen
-Reise & Regionales
-Motorradkleidung
-Porcellana e Ceramica
-Soins de la peau/du visage
-Computer-Komponenten & -Teile
-Music Memorabilia
-?????
-Hand Tools & Equipment
-TV, Video & Home Audio
-Kleinger?te K?che
-Action Figures
-M?nchen & Region
-Damenmode
-Reit- & Fahrsport
-Animation
-Cables & Connectors
-Produktions- & Industriebedarf
-DVDs & Blu-rays
-Odzie? damska
-SCUBA & Snorkelling
-Audio portatifs & ?couteurs
-Kindermode, Schuhe & Access.
-K?rperpflege
-Kids' Clothing, Shoes & Accs
-Antikpuppen
-Odzie? damska
-Bootsport
-Damenmode
-Sewing
-Women's Accessories
-Refrigerators, Freezers
-Rollenspiele & Tabletops
-Elektronik & Elektrotechnik
-Comics
-Motorcycles
-Edelsteine
-Mu?ecas
-?????
-Wohnwagen & Wohnmobile
-Cross Stitch
-Cycling
-Volumi, Artbook e Riviste
-Rasenpflegeger?te
-Herrenmode
-Zubeh?r
-Women's Accessories
-Clothes, Shoes & Accessories
-Brass
-Electrical & Test Equipment
-Echtschmuck
-Massage
-Furniture
-Trading Cards/ CCG
-Dienstleistungen
-Handy- & PDA-Zubeh?r
-Modelleisenbahn Spur H0
-Cz??ci samochodowe
-Kleidung, Schuhe & Accessoires
-Telefony kom?rkowe
-Jewellery Storage & Cleaners
-Cat?gorie test
-Echtschmuck
-Tobakiana
-Date-Lined Ceramics
-????
-Landline Phones
-Komunikacja i ??czno??
-Home Arts & Crafts
-Furniture
-Decorative Arts
-Kleidung, Schuhe & Accessoires
-Basketball
-Animals
-Cz??ci samochodowe
-Software
-Zubeh?r
-C?bles & connecteurs
-TV, Video & Home Audio
-Geschenk- & Werbeartikel
-Meubels en wonen
-Games
-Mp3 & Mp4-spelare
-Zegarki nowych marek
-Cookware, Dining & Bar
-Reise & Regionales
-Malerei
-??/???
-Electrical & Test Equipment
-Weine
-Animals
-Oggetti sportivi
-Gadgets & Other Electronics
-Women's Clothing
-Damenmode
-DVDs & Blu-ray Discs
-Religion & Spirituality
-Handys ohne Vertrag
-????
-Computer-Komponenten & -Teile
-Telefony kom?rkowe
-Radsport
-Postcards
-Kamera- & Fotozubeh?r
-Clothing:GirlToddler
-Transportation
-Test Auctions
-Manufacturing & Metalworking
-Kleidung, Schuhe & Accessoires
-Pozosta?e zegarki na r?k?
-K?nstlerbedarf
-Auto-Tuning & -Styling
-Moto de collection: pi?ces
-Restaurant & service traiteur
-Chitarre e Bassi
-Cz??ci samochodowe: Tuning
-Home Surveillance
-Figuren, Poppetjes & Robots
-Men's Clothing
-Zubeh?r
-Kleidung, Schuhe & Accessoires
-TV- & Heim-Audio-Zubeh?r
-Beads
-Stationery
-Damenmode
-LEGO
-Grafik, Drucke
-DVD, Blu-ray & Heimkino
-Kochen & Genie?en
-Kleidung, Schuhe & Accessoires
-Video Game Consoles
-Cards
-Men's Clothing
-Adult Unisex
-Fu?ball
-Armband- & Taschenuhren
-Furniture
-Fine Jewellery
-Pierres pr?cieuses & Perles
-Damenmode
-Badminton
-Women's Clothing
-Cars & Trucks
-Europa
-Fragrances
-Wedding Supplies
-Kindermode, Schuhe & Access.
-Laufwerke & Speichermedien
-Handcrafted & Finished Pieces
-Brett- & Gesellschaftsspiele
-Latin America
-Software
-Slot Cars
-Men's Shoes
-Fan Apparel & Souvenirs
-Serviettes papier
-iPod- & MP3-Player-Zubeh?r
-Antikspielzeug
-Handy- & PDA-Zubeh?r
-Zubeh?r
-Antique Furniture
-Games
-Kochen & Genie?en
-Vintage Computing
-Drives, Storage & Blank Media
-CDs
-Sewing/ Fabric/ Textiles
-Cardmaking & Scrapbooking
-RC-Modellbau
-Teppiche & Teppichb?den
-Damenschuhe
-Staubsauger
-Vintage Clothing & Accessories
-Indoor Games
-Auto-Ersatz- & -Reparaturteile
-Tekstylia domowe
-Mobile Phones
-Kleidung, Schuhe & Accessoires
-Needlecrafts & Yarn
-Bettwaren, -w?sche & Matratzen
-Skanery
-Professional Broadcasting
-Echtschmuck
-Auto-Tuning & -Styling
-Cz??ci motocyklowe
-Sport
-Accessori cellulari e palmari
-M?bel
-Herrenmode
-M?bel
-DVDs & Blu-rays
-TV, Video & Home Audio
-Women's Clothing
-Fernseher
-Games
-Grafik, Drucke
-Golf
-Fu?ball-Fanshop
-Health Care
-Garten- & Gew?chsh?user
-Zubeh?r & Hardware
-DVDs & Blu-rays
-Home Improvement
-Budownictwo
-Objektive & Filter
-Logiciels
-Handy-Zubeh?r
-Impresoras y esc?neres
-Computer-Komponenten & -Teile
-Bielizna damska,st. k?pielowe
-Kleidung, Schuhe & Accessoires
-Echtschmuck
-Volkskunst
-Art. ko?cielne i dewocjonalia
-Digitalkameror
-Men's Jewelry
-Kindermode, Schuhe & Access.
-Unisex Clothing, Shoes & Accs
-DIY Tools
-Locaux: mat?riel, fournitures
-Hilfsmittel
-V?tements, accessoires vintage
-Antique Furniture
-Kindermode, Schuhe & Access.
-Damenmode
-Damenmode
-Software
-GPS & In-Car Technology
-Clothes, Shoes & Accessories
-Luggage & Travel Accessories
-Wand & Boden
-Fancy Dress & Period Costume
-Loisirs, vie pratique
-CDs
-Cards
-Bi?uteria z?ota
-Home Improvement
-Pi?ces - ?chelle mondiale
-Watches
-Weitere Uhren
-Habits Sp?cifiques
-Kinderkleding- jongens
-Color TVs
-Zigarren & Tabakwaren
-Auto-Ersatz- & -Reparaturteile
-Holzspielzeug
-Budownictwo i akcesoria
-Advertising
-Obuwie m?skie
-Football Shirts
-Auto-Ersatz- & -Reparaturteile
-Disneyana
-iPads, Tablets, eReaders
-Voeding
-Heimnetzwerk & Zubeh?r
-Woodcrafts
-Books & Manuscripts
-Vehicle Electronics & GPS
-Education & Professional
-Software
-Silver
-Appliances
-Hunting
-Home Improvement
-Decorative Collectibles
-Historische Wertpapiere
-Vintage & Antique Jewellery
-Geschenk- & Werbeartikel
-Welt der Tiere
-Haushalt
-DVDs & Blu-rays
-Meubles, d?coration, Xx?me
-Motorrad- & Kraftradteile
-Rasenm?her
-Golf
-Monitors, Projectors & Accs.
-Autre
-Handys mit Vertrag
-Music Memorabilia
-Fashion Jewellery
-Cell Phone Accessories
-??/??/??
-Reise & Regionales
-Basteln & Kreativit?t
-Auto-Ersatz- & -Reparaturteile
-Other
-Bath & Body
-Office
-Lecteur, stockage, sup. vierge
-Computer Components & Parts
-Donna: Scarpe
-Camera & Photo Accessories
-Medizin & Labor
-Accommodation
-Paper Money: World
-PC- & Videospiele
-Clothes, Shoes & Accessories
-Dekoration
-Decorative Collectibles
-Video Games
-Cz??ci samochodowe
-Photographie argentique
-Emballage & exp?dition
-Reclame- & Caf?voorwerpen
-Cartes de jeux de collection
-Video, Film
-Parts & Accessories
-Handys ohne Vertrag
-Automobilia
-Cell Phone Accessories
-Kaffee & Espressomaschinen
-Haushalt
-Damen-Accessoires
-Cars
-Camera & Photo Accessories
-Skin Care
-Gommoni
-Odzie? m?ska
-Hunting
-?bernachtungen
-Electrical & Test Equipment
-Prints
-Archery
-Comics
-Television Memorabilia
-Banknotes
-Grafik, Drucke
-??????
-Reise & Regionales
-Cell Phones & Smartphones
-Non-Fiction
-Damenbekleidung
-Auto-Tuning & -Styling
-Bottles, Jars, Jugs
-Kindermode, Schuhe & Access.
-Weitere Sammelgebiete
-Machinerie lourde
-
-Software
-Damenmode
-Ordinateur: composants, pi?ces
-Echtschmuck
-??
-Kleinger?te K?che
-Luxus-Accessoires
-Baby Books
-Folkloreschmuck
-Test Auctions
-V?tements pour femmes
-TV, Video & Home Audio
-Handcrafted, Artisan Jewelry
-Mobile Phones
-B?rotechnik
-Zubeh?r & Hardware
-Land
-Fournitures tout usage
-Drives, Storage & Blank Media
-Katzen
-Skisport & Snowboarding
-Modelleisenbahn Spur H0
-P??ki i rega?y
-Outdoor Sports
-Rhein-Neckar
-Software
-Men's Accessories
-Computer-Komponenten & -Teile
-Basteln & Kreativit?t
-TV, Video & Home Audio
-Ethnic, Regional & Tribal
-Baby & Toddler Clothing
-Model Trains
-DVD, Blu-ray & Heimkino
-Accessoires pour femmes
-Vintage & Antique Jewellery
-Schule & Ausbildung
-Bootonderdelen en -accessoires
-Militaria
-Cz??ci samochodowe
-Figuren
-Non-Fiction
-Perles, Fabrication de bijoux
-Reise & Regionales
-Vinilos
-Commonwealth/ British Colonial
-Music Memorabilia
-Horse Racing Memorabilia
-Sonnenschutz & Solarium
-Piercings
-Antiquities
-Back?fen & Herde
-Kindermode, Schuhe & Access.
-Trading Cards
-Kleinger?te K?che
-??????/??
-Skin Care
-Postcards
-R?seau domicile, connectivit?
-Percussion
-Accesorios para m?viles y PDAs
-Objets de Disney
-Camping, Hiking
-Cards
-Herren-Designerbekleidung
-Xbox 360
-Parts & Accessories
-Car, Truck Parts
-Sp?cialit?
-Records
-Cards
-Fitness
-Group Buy Listings
-Cartes de collection
-Cartes de collection
-Hunting
-Automobile
-Sachb?cher & Ratgeber
-Accessoires ordinateur
-Computer Components & Parts
-Personenauto's
-??????/??
-Echtschmuck
-Cars
-Fotografie
-Deutschland
-Audio Books
-Frankrijk
-World Coins
-Make-up
-????
-Woodwind
-Outdoor Sports
-Fragrances
-Veranstaltungs- & DJ-Equipment
-Olympic Memorabilia
-Loose Beads
-Tennis
-Zubeh?r
-Festivals & Konzerte
-Music Memorabilia
-V?tements & souvenirs de fan
-Tobacciana
-Men's Jewellery
-Handy- & PDA-Zubeh?r
-Glass
-Auto-Hi-Fi & Navigation
-Games
-Glass
-Advertising
-Lingerie & Strandkledij
-Versandmaterialien
-Cross Stitch
-Grafik, Drucke
-Diecast & Vehicles
-Nederland
-Home Arts & Crafts
-Scuba, Snorkeling
-Lampen & Licht
-Femmes: V?tements
-Abbigliamento Moto
-Echtschmuck
-Parts & Accessories
-Pocket Watches
-Auto-Ersatz- & -Reparaturteile
-Baby & Toddler Clothing
-Test Auctions
-Gastro & Nahrungsmittelgewerbe
-Yard, Garden & Outdoor Living
-?ducatifs
-Silver
-K?nstlerbedarf
-Major Appliances
-Cricket
-Damenmode
-Veranstaltungs- & DJ-Equipment
-Fu?ball
-Music Memorabilia
-Mobile Phones
-TV, Video & Home Audio
-GPS Accessories
-Romans, litt?rature
-Gro?handel & Sonderposten
-Yard, Garden & Outdoor Living
-Software
-Gadgets & Other Electronics
-Brett- & Gesellschaftsspiele
-Damenmode
-String
-Handys ohne Vertrag
-Festivals & Konzerte
-Asia
-Dekoration
-Dolls
-Hochzeit & Besondere Anl?sse
-Fabric/ Textiles
-Parts & Accessories
-Weine
-Handys ohne Vertrag
-Handys mit Vertrag
-Transport
-Cz??ci motocyklowe
-Action- & Spielfiguren
-?
-Kids' Clothing, Shoes & Accs
-Objektive & Filter
-Cz??ci samochodowe
-Skisport & Snowboarding
-Sewing & Fabric
-Cz??ci motocyklowe
-Mobiliar & Interieur
-Stamping & Embossing
-CDs
-Zubeh?r
-Katzen
-DVD - Spectacles musicaux
-Photographic Images
-Beads & Jewelry Making
-Revues, manuels, catalogues
-Breweriana
-Cars
-Deutschland
-TV, Movie & Character Toys
-DVD, Blu-ray & Heimkino
-Outdoor Sports
-Fu?ball
-Praktisch & Educatief
-Other
-Cell Phone Accessories
-Handy- & PDA-Zubeh?r
-Motorrad- & Kraftradteile
-Fashion Jewelry
-Art, Architecture & Design
-Yard, Garden & Outdoor Living
-Kurzreisen
-D?coration de maison
-Voiliers
-Men's Jewellery
-Papiergeld Deutschland
-Modeschmuck
-Profumi
-Computer Components & Parts
-Automobiles & camions
-UMDs
-Auto-Tuning & -Styling
-Motorrad- & Kraftradteile
-Kurzreisen
-Kurzreisen
-IT, Communication
-K?nstlerbedarf
-Laptop & Desktop Accessories
-K?rperpflege
-Parts & Accessories
-Cleaning, Housekeeping
-Biographies & Autobiographies
-Transport
-M?nzen International
-Video Games
-Home Decor
-Telefony kom?rkowe
-Kleidung, Schuhe & Accessoires
-Advertising
-Esoterik, Mystik & Magie
-PC's
-Handys mit Vertrag
-Parfums
-Bath
-Handy- & PDA-Zubeh?r
-Girl's Clothing
-Conexi?n de redes
-Handys ohne Vertrag
-Antikschmuck
-Dienstleistungen
-Europa
-Poterie & porcelaine
-Kleidung, Schuhe & Accessoires
-Modelleisenbahn
-????
-Porcelain/ China
-Back?fen & Herde
-Funktechnik
-Records
-Canada
-Car, Truck Parts
-Zeitschriften
-Glass
-Advertising
-Kindermode, Schuhe & Access.
-Auto-Ersatz- & -Reparaturteile
-AGD
-Jeux
-Metaphysical & New Age
-Militaria
-Kolekcje sportowe
-Haushalt & K?che
-Damenmode
-Bijouterie de qualit?
-Bettausstattung
-Laufwerke & Speichermedien
-???
-Odzie? damska
-Computer-Komponenten & -Teile
-Sicherheit
-Computer Components
-Lingerie
-Couture & tissus
-Mobile Phones
-Maquillage/cosm?tiques
-Motorrad- & Kraftradteile
-Sachb?cher & Ratgeber
-Damesschoenen
-???????
-Non-Fiction
-Damenmode
-Kindermode, Schuhe & Access.
-Bad- en strandmode
-Motorcycles
-Mobile Accessories
-Deutschland
-Music Memorabilia
-Heavy Equipment Parts & Accs
-Domotique
-Puzzles
-Handys ohne Vertrag
-Vehicle Electronics & GPS
-Fan Apparel & Souvenirs
-Mini Moto Ricambi
-Vintage & Antique Jewellery
-Festnetztelefone & Zubeh?r
-??
-Computer-Komponenten & -Teile
-Handys ohne Vertrag
-Herren-Accessoires
-Gadgets & Other Electronics
-V?gel
-D?tail & services
-Varie
-??/??/??
-Restaurant & Catering
-Bootonderdelen en -accessoires
-Baby & Toddler Clothing
-Cars
-Ansichtkaarten en foto's
-Parfums
-Kleinger?te K?che
-Puppen
-Computer-Komponenten & -Teile
-Cars
-Automobilia
-Vintage & Antique Jewelry
-Decorative Collectibles
-??????/??
-Outdoor Sports
-Herenhorloges
-Luggage Accessories
-Peripherals
-Fantasy
-Glass & Mosaics
-Mode Fille
-Other Sound & Vision
-Mobile Phones
-Zestawy odzie?y
-Cultures & Ethnicities
-Cz??ci samochodowe: Tuning
-Reisekoffer & -taschen
-Mode B?b? Mixte
-Vintage & Antique Jewellery
-Mat?riel param?dical
-GPS & In-Car Technology
-Bettwaren, -w?sche & Matratzen
-Damenmode
-??
-Transportation
-Studium & Wissen
-Banknotes
-Educational Toys
-Damenmode
-Paintings
-Livres Toutes Disciplines FR
-Auto-Ersatz- & -Reparaturteile
-Agrar, Forst & Kommune
-Stoneware
-Home Improvement
-Fiction
-Artist Jewelry
-Skin Care
-Art from Dealers & Resellers
-Serien & Lizenzprodukte
-Lighting, Fans
-Lecteur, stockage, sup. vierge
-Teleskope & Ferngl?ser
-Dolls
-Gutscheine
-V?tements pour hommes
-Laufwerke & Speichermedien
-Cultures & Ethnicities
-Fu?ball-Fanshop
-Security
-Music Memorabilia
-Software
-Computer Components & Parts
-Cuisine, restaurants & bar
-Spielzeugautos
-Other
-Animals
-Profumi
-Musik & Instrumente
-Echtschmuck
-Home Improvement
-R?seau d'entreprise, serveurs
-Games
-Modelleisenbahn
-Papier- en schrijfwaren
-Zubeh?r & Hardware
-Exercise & Fitness
-Home Arts & Crafts
-Papier, B?ro- & Schreibwaren
-Varie
-Glass
-Decorative Collectibles
-Damenmode
-Biographies
-Mobile Phones
-Fragrances
-Indian Coins
-Parfums
-Motorrad- & Kraftradteile
-Instrument Accessories
-Games
-Couture, Broderie, Tricot
-Bigiotteria
-Laptop Accessories
-Other
-Militaria
-Armband- & Taschenuhren
-Wholesale, Large & Small Lots
-Reise & Regionales
-Maternity/ Pregnancy
-Objektive & Filter
-C?bles & connecteurs
-Hardware
-Feste & Besondere Anl?sse
-TV, Movie & Character Toys
-Dameskleding
-Wholesale Lots
-Movie Memorabilia
-Sauna & Schwimmbad
-Damen-Accessoires
-Handy- & PDA-Zubeh?r
-Kunst & Kultur
-Gry
-Motorrad- & Kraftradteile
-%''%K%a!<%7%g%s
-??/?????
-Golf
-Electrical & Test Equipment
-Autoaccessoires en onderdelen
-Luxus-Accessoires
-Kinderm?bel & Wohnen
-Pro Audio Equipment
-Television Memorabilia
-Produktions- & Industriebedarf
-Babykleding - unisex
-Herrenmode
-Auto-Tuning & -Styling
-Damenmode
-Gewerbe
-Radio-Controlled
-Nail Care
-Laufwerke & Speichermedien
-CDs
-Women's Clothing
-Mobile Phones
-Kleidung, Schuhe & Accessoires
-Heim-Audio & HiFi
-Cards
-Fashion Jewellery
-Laptop & Desktop Accessories
-Animation Art & Characters
-Football Shirts
-P?yty kompaktowe
-Baby & Toddler Clothing
-Baby & Toddler Clothing
-Publicit?
-Damenmode
-Stage Lighting & Effects
-V?tements & souvenirs de fan
-Computer Components & Parts
-Pi?ces & access. de machinerie
-Rollenspiele & Tabletops
-Cartes de collection
-Automobile
-Computer Components & Parts
-Tickets & Experiences
-DIY Tools
-Dekoration
-Auto-Ersatz- & -Reparaturteile
-Ansichtskarten
-TV-Receiver & Set-Top-Boxen
-Music Memorabilia
-Damenmode
-Transportation
-Furniture
-Light Equipment & Tools
-Women's Clothing
-Holiday & Seasonal
-Maquettes
-Auto-Ersatz- & -Reparaturteile
-Computer Components & Parts
-Jeux de soci?t?
-Damenmode
-Vehicle Electronics & GPS
-Tools
-Yard, Garden & Outdoor Living
-Auto - Tuning & Styling
-Motocyclettes
-Women's Clothing
-Cars
-Damenmode
-Kitchen & Home
-Accessories
-Wholesale, Large & Small Lots
-Ride-On Toys
-Cartes de collection
-Heim-Audio & HiFi
-B?rotechnik
-Fische & Aquarien
-Composants: TV, son
-Education & Professional
-Weitere Sportarten
-Fine Jewellery
-Tijdschriften en kranten
-Fantasy/ Myth/ Magic
-iPods & MP3-Player
-Historische Wertpapiere
-Scolaire, dictionnaires
-Teich- & Bachlaufsysteme
-Cycling
-Music Memorabilia
-Fu?ball
-Guitar
-Armband- & Taschenuhren
-Restaurants
-Musicassette
-??????
-DVD, Blu-ray & Heimkino
-Pi?ces & accessoires
-Enterprise Networking, Servers
-Trading Cards/ CCG
-Enterprise Networking, Servers
-Armband- & Taschenuhren
-Women's Clothing
-Percussions
-Weitere Ballsportarten
-Historische Baustoffe
-Greeting Cards & Party Supply
-Pianos, Keyboards & Organs
-Women's Accessories
-Kleidung, Schuhe & Accessoires
-Echtschmuck
-Pottery
-?berraschungseier
-Science Fiction & Horror
-Bureau
-Uomo: Scarpe
-M?bel
-Pottery & China
-Reise & Regionales
-Handy- & PDA-Zubeh?r
-Kochen & Genie?en
-Wedding Supplies
-Angelsport
-Motorcycle Parts & Accessories
-Asian Antiques
-Hilfsmittel
-Loisirs, vie pratique
-Cromos
-Medikamente von Apothekern
-Motorrad- & Kraftradteile
-Cyclisme
-Kitchen, Dining & Bar
-Manga
-M?bel
-Flags
-Echtschmuck
-Metalworking/ Milling/ Welding
-Beleuchtung
-Revues, manuels, catalogues
-Guitar
-Auto-Ersatz- & -Reparaturteile
-Bath
-Famille des cuivres
-Souvenirs musicaux
-Commonwealth/ British Colonial
-Fische & Aquarien
-Porzellan & Keramik
-Decorative Arts
-Motorrad- & Kraftradteile
-Sicherheit
-Militaria
-TV, Video & Home Audio
-Damenmode
-Digitalkameras
-Handcrafted, Artisan Jewelry
-F?tes & saisonniers
-Women's Clothing
-????
-Crafts & Hobbies
-?bernachtungen
-Echtschmuck
-Rollenspiele & Tabletops
-Women's Clothing
-Wand & Boden
-Damenschuhe
-Needlecrafts & Yarn
-Fragrances
-??/??
-??CD
-Meubles
-Kleinger?te K?che
-Mat?riel ?lectrique & d'essai
-Vehicle Electronics & GPS
-Trading Cards
-Cars
-Cavi e connettori
-Computer-Komponenten & -Teile
-Healthcare, Lab & Life Science
-Sewing (1930-Now)
-Videojuegos
-B?rotechnik
-Software
-Streekromans
-Kurzreisen
-Bijoux de mode
-Motorcycles
-Sports d'hiver
-Weine
-Parts & Accessories
-FMCG
-Militaria
-Sports d'?quipe
-Kurzreisen
-Tourisme
-Srebra
-Digital Cameras
-Golf
-Information Products
-Shaving & Hair Removal
-Auto-Anbau- & -Zubeh?rteile
-Women's Clothing
-Science fiction & horreur
-MP3 Player Accessories
-Model Trains
-Cycling
-Obuwie m?skie
-Heimnetzwerk & Zubeh?r
-Home Furnishings
-Mobile Phone & PDA Accessories
-Quilting
-Arts Graphiques
-Dolls
-Computer-Komponenten & -Teile
-Sachb?cher & Ratgeber
-Litt?rature & Fiction NL
-Beads & Jewelry Making
-Computers
-Herrenmode
-Auto-Anbau- & -Zubeh?rteile
-??
-Plakate & Kunstdrucke
-Wand & Boden
-Parts & Accessories
-TV, Video & Home Audio
-Magazines
-Cz??ci samochodowe
-Kleidung, Schuhe & Accessoires
-A/V Accessories & Cables
-Damenmode
-Fragrances
-Philately/ Postal History
-M?nzen International
-Agrar, Forst & Kommune
-Cars
-Anlageobjekte
-Handys ohne Vertrag
-Hunting
-Outdoor Sports
-V?tements b?b?s, tout-petits
-Baby Bathing & Grooming
-Teleskope & Ferngl?ser
-Gitarren
-Imprimantes, scanneurs & acc.
-Cycling
-Veranstaltungs- & DJ-Equipment
-Comics
-Tragbare Disc-Player & Radios
-Dog Supplies
-Kost?me & Verkleidungen
-Auto-Ersatz- & -Reparaturteile
-????
-Damenmode
-Fragrances
-Laptop & Desktop Accessories
-Manufacturiers & m?tallurgie
-Motorcycle Parts & Accessories
-??/??
-Kinder- & Jugendliteratur
-Gitarren
-Lotti e Stock Libri
-Angelsport
-Skateboarding
-Vintage Movie & Photography
-DVD
-Kleidung, Schuhe & Accessoires
-Herrenschuhe
-Kindermode, Schuhe & Access.
-Trading Cards
-Tauchen
-Clothing, Helmets & Protection
-Motive
-Decorative Collectibles
-Alte Berufe
-Saisonales & Feste
-Mobile Phones
-Odzie? pozosta?a
-Cars
-Cars & Trucks
-Herrenmode
-Guitar
-Orologi da Polso: alla Moda
-Cards
-Werkzeuge
-Antikspielzeug
-Software
-Drucker, Scanner & Zubeh?r
-Vintage & Antique Jewelry
-Elektromaterial
-Kindermode, Schuhe & Access.
-Heimnetzwerk & Zubeh?r
-Militaria
-Caravanas y Remolques
-Kleinger?te Pflege & Wellness
-Verre, cristal
-T?l?phones mobiles
-Parfums
-Motorr?der
-Moviles
-Jeux
-Sporting Goods
-Baugewerbe
-Fitness & Gym
-????/??
-Fitness
-TV & Film Character Toys
-Hobby & K?nstlerbedarf
-Kurzreisen
-Auto-Tuning & -Styling
-Sicherheitstechnik
-Objets de collection de d?cora
-Skin Care
-GPS & In-Car Technology
-Kinderschoenen
-Gitarren
-Hair Care & Styling
-Digitale camera's
-Kindermode, Schuhe & Access.
-Games
-Fu?ball-Fanshop
-Beleuchtung
-Handys mit Vertrag
-Watches
-Modeschmuck
-Baugewerbe
-Makeup
-M?quinas recreativas
-Notebook- & Desktop-Zubeh?r
-Pi?ces & accessoires
-Fotografie & Fotokunst
-Audio portatifs & ?couteurs
-MRE, fournitures industrielles
-Printers, Scanners & Supplies
-Bijoux d'?poque, anciens
-Writing Instruments
-Baugewerbe
-Lernspielzeug
-Claviers, souris & pointeurs
-Zubeh?r
-Lampes, plafonniers-?ventails
-Damenmode
-Kleidung, Schuhe & Accessoires
-Fiction Books
-Baby & Toddler Clothing
-Cars & Trucks
-Accessoires de cellulaires
-Car Seats & Accessories
-Reiseaccessoires
-Outdoor Sports
-Motorrad- & Kraftradteile
-Robots, Automates
-Wholesale Lots
-Boy's Clothing
-Soins de sant?, sciences vie
-Vehicle Electronics & GPS
-V?tements b?b?s, tout-petits
-Women's Clothing
-Cycling
-Advertising
-Druckerei & Copyshop
-Herrenschuhe
-Damenmode
-Feeding
-Cars & Trucks
-Camping & Outdoor
-Laptop & Desktop Accessories
-Restaurant & Catering
-Fashion Jewellery
-Laufwerke & Speichermedien
-Laufwerke & Speichermedien
-??/??
-TV, Video & Home Audio
-Animals
-Oferty hurtowe i wyprzeda?e
-Meubles
-Men's Clothing
-Glass
-Computer-Klassiker
-Games
-Cartes de jeux de collection
-Kindermode, Schuhe & Access.
-CDs
-Toys & Activities
-Damenmode
-Kochen & Genie?en
-Skisport & Snowboarding
-Souvenirs militaires
-??GSM??
-Test Auctions
-Echtschmuck
-Wholesale Lots
-Auto-Ersatz- & -Reparaturteile
-Budownictwo
-Hard Disk Drives
-VHS-video?s
-Pi?ces d?tach?es
-????
-Plants/ Seeds/ Bulbs
-Science & Medicine (Pre-1930)
-Sun Protection & Tanning
-????
-Audio Books
-Damenmode
-Nonfiction
-Sonstige Branchen & Produkte
-Damenmode
-Damenmode
-Pens & Writing Equipment
-AGD
-??
-Antikspielzeug
-Agriculture
-Advertising
-Dierenvoeding
-Design & Stil
-Trains/ Railway Models
-Cat Supplies
-Holzbearbeitung & Tischlerei
-Videojuegos
-Gastronomie & F?tes
-Antyki, sztuka mi?dzynarodowa
-Gastro & Nahrungsmittelgewerbe
-Militaria
-??????
-Action- & Spielfiguren
-Kamera- & Fotozubeh?r
-??CD
-Auto-Ersatz- & -Reparaturteile
-PC- & Videospiele
-Music Memorabilia
-Heim-Audio & HiFi
-KFZ-Handwerk
-Parfums
-Accesorios c?maras/fotograf?a
-Poppetjes en figuurtjes
-Middle East
-Gesichtspflege
-Festivals & Konzerte
-Movie Memorabilia
-Ponds & Water Features
-Car Manuals & Literature
-Cz??ci samochodowe
-Cycling
-Kids' Clothing, Shoes & Accs
-Computer-Komponenten & -Teile
-Fournitures pour chien
-Produktions- & Industriebedarf
-Boats Parts & Accessories
-Damenmode
-Memorabilia
-Diecast & Vehicles
-Geschirrsp?ler
-Fan Apparel & Souvenirs
-Acciones y Papeles
-Diamond Jewellery
-Skisport & Snowboarding
-Herrenschuhe
-Football Shirts
-Parts & Accessories
-Bath & Body
-Software
-Bricolage
-Motorrad- & Kraftradteile
-Grafik, Drucke
-Men's Wrist Watches
-Festnetztelefone & Zubeh?r
-Cards
-Dischi Vinile 33 giri - 12''
-B?rotechnik
-Pens & Writing Equipment
-Cigarette/ Tea/ Gum Cards
-Pflanzen, B?ume & Str?ucher
-Scrapbooking
-T?
-Hochzeit & Besondere Anl?sse
-Loose Diamonds & Gemstones
-Damentaschen
-Videocassette e VHS
-Damenmode
-Festivals & Konzerte
-Yard, Garden & Outdoor Living
-Children & Teens
-Guitars
-TV- & Heim-Audio-Zubeh?r
-Video Games
-Damenmode
-Movie Memorabilia
-Computer Components & Parts
-Handy-Attrappen
-Loose Diamonds & Gemstones
-Armband- & Taschenuhren
-Instruments
-Enterprise Networking, Servers
-Education & Professional
-Trading Cards
-Tee
-Hair Removal & Shaving
-Computer-Komponenten & -Teile
-Perfumy
-Auto-Anbau- & -Zubeh?rteile
-Light Equipment & Tools
-TV-Receiver & Set-Top-Boxen
-Autographs-Original
-Herrenmode
-Veranstaltungs- & DJ-Equipment
-Figurines, Statues
-TV- & Heim-Audio-Zubeh?r
-Ricambi Auto
-Football Memorabilia
-Guitare
-Animation Art & Characters
-Homeware, Kitchenware
-B?rotechnik
-Livres de bandes dessin?es
-Motor Sport Memorabilia
-Coins: US
-Cz??ci motocyklowe
-Cat Supplies
-Maps, Atlases & Globes
-Makeup
-Women's Clothing
-Casino
-Pi?ces France
-Fan Apparel & Souvenirs
-Livres de b?b?s & faire-part
-Computer-Komponenten & -Teile
-Reise & Regionales
-Ansichtskarten
-M?nzen Europa
-Cuisine: Arts de la table
-Woodwind
-Emballage & exp?dition
-K?nstlerbedarf
-Tragbare Disc-Player & Radios
-???????
-Notebook- & Desktop-Zubeh?r
-Versandmaterialien
-MRO & Industrial Supply
-Cultures & Ethnicities
-Fan Apparel & Souvenirs
-Fanartikelen
-Handy- & PDA-Zubeh?r
-Car Seats, Accessories
-Women's Clothing
-Chaussures de femme
-Deodorants
-?????/??
-Zubeh?r & Hardware
-Odzie? m?ska
-Clothing, Merchandise, Media
-M?bel
-Dollhouse Miniatures
-Parfum, eau en after shave
-Damenmode
-T?l?, vid?o & audio domestique
-Kindermode, Schuhe & Access.
-DVDs & Blu-rays
-Kleinger?te Pflege & Wellness
-Auto-Hi-Fi & Navigation
-Cars
-Monitore, Projektoren & Zub.
-Hobby & K?nstlerbedarf
-Casino
-Sammleruhren
-Cards
-Pens & Writing Equipment
-Ruilkaartspellen
-H?rb?cher & H?rspiele
-Babyspeelgoed
-Kurzreisen
-Electrical
-Piercing-/K?rperschmuck
-Radio, Gramophone, Phone
-Action figures e statiche
-Diecast & Vehicles
-Elektromaterial
-Maquettes
-Echtschmuck
-Pottery
-Kindermode, Schuhe & Access.
-Fan Apparel & Souvenirs
-Vintage
-Parts & Accessories
-Porte-clefs
-Cartes postales
-Photocopier Accessories, Parts
-Art?culos de Escritorio
-Drives, Storage & Blank Media
-Polizei & Beh?rden
-Nutrition
-Souvenirs & Travel Memorabilia
-Nautika & Maritimes
-Tabak, Feuerzeuge & Pfeifen
-Metallbearbeitung & Schlosser.
-Heavy Equipment Parts & Accs
-Kitchen, Dining & Bar
-Handys mit Vertrag
-Mat?riel ?lectrique & d'essai
-Sonstige
-Winter Sports
-Kleidung, Schuhe & Accessoires
-Welt der Tiere
-Kleidung, Schuhe & Accessoires
-Analoge Fotografie
-Num?riques - Accessoires
-Tools
-Motorrad- & Kraftradteile
-Zubeh?r
-Ricambi Auto d'Epoca
-Baustoffe
-Papier, B?ro- & Schreibwaren
-Armband- & Taschenuhren
-Parts & Accessories
-Drives, Storage & Blank Media
-Fishing
-Relojes de Pulsera
-Autoaccessoires en onderdelen
-Meble designerskie
-Movie Memorabilia
-Memorabilia
-Analoge Fotografie
-Klassische Konsolen
-Film Photography
-Modelleisenbahn Spur H0
-Parts & Accessories
-Tv & Homecinema
-Fine Jewellery
-Preschool Toys & Pretend Play
-TV- & Heim-Audio-Zubeh?r
-Weitere Sportarten
-Bettausstattung
-??
-Auto-Ersatz- & -Reparaturteile
-Handys ohne Vertrag
-Jogging
-Auto-Ersatz- & -Reparaturteile
-Gitarren
-Damenmode
-Mode Homme
-Vintage & Antique Jewellery
-Records
-Desktops & All-In-Ones
-Damenmode
-CD's
-Swimming
-Przemys? tekstylny i pokrewne
-TV, Video & Home Audio
-Pi?ces & accessoires
-Memorabilia
-Baby & Toddler Clothing
-Kinderfahrzeuge
-Notebook- & Desktop-Zubeh?r
-Car, Truck Parts
-Auto-Tuning & -Styling
-Jardin, Ext?rieur
-Accessoires de cellulaires
-Audio portatifs & ?couteurs
-Internationale Antiq. & Kunst
-Printers, Scanners & Supplies
-Notebook- & Desktop-Zubeh?r
-Vinyles
-Jeux
-Guitar
-Auto-Ersatz- & -Reparaturteile
-Papiergeld Welt
-Fantasy, Mythical & Magic
-Fu?ball
-Fournitures tout usage
-Telefony kom?rkowe
-Non-Fiction
-Tastaturen, M?use & Pointing
-Wholesale Lots
-Reise & Regionales
-Klussen
-Auto-Ersatz- & -Reparaturteile
-Modelleisenbahn
-Polizei & Beh?rden
-TV- & Heim-Audio-Zubeh?r
-Pi?ces - Canada
-Maternity Clothing
-Postcards
-Dolls
-Verre
-Computer-Komponenten & -Teile
-Tickets & Toegangskaarten
-Stamps
-Damenmode
-Food & Beverages
-Monitore, Projektoren & Zub.
-iPods & MP3-Player
-Cars & Trucks
-Fan Apparel & Souvenirs
-Fashion Jewellery
-Klimaanlagen & Heizger?te
-Golf
-Weitere Ballsportarten
-Diecast & Vehicles
-Music Memorabilia
-Wholesale Lots
-Travaux de r?novation
-????
-Silver
-Video Games
-Fragrances
-Kabel & Steckverbinder
-Desktops & All-in-One-PCs
-Digitalkameras
-Gro?handel & Sonderposten
-Mobile Phones
-Livres Toutes Disciplines NL
-Kids' Clothes, Shoes & Accs.
-Kochen & Genie?en
-Elektrisches Spielzeug
-Lamps, Lighting & Ceiling Fans
-Kochen & Genie?en
-Sewing & Fabric
-??
-Kurzreisen
-Antykwariat
-Birofilistyka
-Other Formats
-Homeware, Kitchenware
-Femmes: V?tements
-Motorrad- & Kraftradteile
-Deutschland
-Parts & Accessories
-Art du Monde
-Haushalt
-Beads
-Shaving & Hair Removal
-Coches: repuestos
-H?rb?cher & H?rspiele
-Glass
-Cell Phone Accessories
-Coffee Makers (Automatic)
-Books & Manuscripts
-Kitchenalia
-Non-Fiction
-Weitere Sportarten
-Holiday & Seasonal
-Test Auctions
-Articoli per radioamatori
-Herrenschuhe
-Inlineskating
-iPod- & MP3-Player-Zubeh?r
-Fitness
-Fan Apparel & Souvenirs
-Personnages de t?l? & de films
-Manufacturing & Metalworking
-Cards
-Equip. y Material Oficinas
-Zubeh?r
-Kleinger?te Haushalt
-Needlecrafts & Yarn
-Trading Cards
-Kindermode, Schuhe & Access.
-Parts & Accessories
-Clothing, Helmets & Protection
-Home Furnishings
-Accessories
-Industrial Tools
-B?b?-Fille
-Equestrian
-Deutschland
-Bi?uteria do piercingu
-Baby Health & Safety
-Lotes de Cine, DVD y Pel?culas
-Kids' Clothes, Shoes & Accs.
-Kindermode, Schuhe & Access.
-Retail & Services
-Utensili da modellismo
-Restaurant & service traiteur
-Computer-Komponenten & -Teile
-V?tements & souvenirs de fan
-Damenschuhe
-Produktions- & Industriebedarf
-Home Furniture
-Historical Memorabilia
-Vacances
-Decorative Collectibles
-Cars & Trucks
-Kleidung, Schuhe & Accessoires
-Analoge Fotografie
-Cards
-Sheet Music & Song Books
-Auto-Ersatz- & -Reparaturteile
-???????
-Polskie banknoty
-Other Crafts
-Cz??ci samochodowe
-Loose Diamonds
-Herren-Designer-Accessoires
-Autographs
-????
-Romanhefte
-Cz??ci samochodowe: Tuning
-Auto-Ersatz- & -Reparaturteile
-Auto-Ersatz- & -Reparaturteile
-Software
-Health Care
-Manufacturing & Metalworking
-Action Figures
-Souvenirs musicaux
-Autres Nintendo
-Model Railroads & Trains
-Pi?ces Euro
-Magazine Back Issues
-Motorcycle Parts & Accessories
-Articles religieux
-Woodwind
-Verschonen & Verzorgen
-Kindermode, Schuhe & Access.
-Radsport
-Cz??ci samochodowe
-Baby & Toddler Clothing
-?lectronique, pile, remontoire
-Cycling
-V?tements pour enfants
-Snooker & Pool
-Sony Consoles
-Echtschmuck
-Tuin & Terras
-Electrical & Test Equipment
-TV series en programma's DVD's
-Comics
-Body Jewellery
-Bad & K?che
-Elettronica ed Elettricit?
-Baby Clothing
-?????/??
-Baby & Toddler Clothing
-??/??
-Cars
-Mat?riel de radiocommunication
-Music
-Radio Communication Equipment
-Back?fen & Herde
-Diapering
-Science & Medicine (Pre-1930)
-Kurzreisen
-Auto-Ersatz- & -Reparaturteile
-Equitation
-Kurzreisen
-Radsport
-Homeware, Kitchenware
-Bullion
-Gardening
-V?tements pour femmes
-CD
-Animation
-Studium & Wissen
-Radio Communication Equipment
-Malerei
-Software
-Cartes de collection
-Music Memorabilia
-Healthcare, Lab & Life Science
-Furniture
-Sicherheitstechnik
-Sport
-Auto-Anbau- & -Zubeh?rteile
-Digitalkameras
-Asian Antiques
-Auto-Kindersitze & Zubeh?r
-Handys mit Vertrag
-Technik & Ger?te
-iPods & MP3-Player
-Appliances
-Echtschmuck
-Auto-Anbau- & -Zubeh?rteile
-Damenmode
-Gesichtspflege
-Auto-Ersatz- & -Reparaturteile
-Fan Apparel & Souvenirs
-Handys ohne Vertrag
-Bistrot
-Het aanzien van...
-Natural & Alternative Remedies
-International
-Garten- & Gew?chsh?user
-Non-Fiction
-Trading Cards
-R?seau d'entreprise, serveurs
-Beanbag Plush
-Computer-Komponenten & -Teile
-V?tements & souvenirs de fan
-Kfz-Services & -Reparaturen
-????
-Klein- & Nagetiere
-VHS Tapes
-Damenmode
-Cellulari e smartphone
-Philately/ Postal History
-Packing & Posting Supplies
-Feste & Besondere Anl?sse
-Damenmode
-Herrenschuhe
-Cars
-Asian Art
-????
-Pi?ces & accessoires
-DVD, Blu-ray & Heimkino
-Scrapbooking & Paper Crafts
-Kindermode, Schuhe & Access.
-Advertising
-R?seau d'entreprise, serveurs
-Equipment
-Damenmode
-Televisori e Proiettori
-Dessins anim?s & personnages
-Computer-Komponenten & -Teile
-Damenmode
-Auto - Tuning & Styling
-Fashion Jewellery
-Journaux, revues, magazines
-Printers, Scanners & Supplies
-Papiergeld Deutschland
-Stroje ?lubne
-Console
-Telefony kom?rkowe
-Damenschuhe
-??????????????????
-Comics
-Software
-Non-Fiction Books
-Claviers, souris & pointeurs
-Coins
-Auto-Ersatz- & -Reparaturteile
-Installation
-Handy- & PDA-Zubeh?r
-Women's Clothing
-Fan Apparel & Souvenirs
-Mobile Phones
-Kochen & Genie?en
-M?nzen ?sterreich
-Souvenirs & Travel Memorabilia
-Manufacturing & Metalworking
-Skisport & Snowboarding
-Vintage & Antique Jewellery
-Building Materials, DIY
-Perfumes & Fragrances
-Medizin & Labor
-Dolls
-Souvenirs ?missions t?l?vis?es
-Bettwaren, -w?sche & Matratzen
-Motorrad- & Kraftradteile
-Food & Beverages
-Cuisine, restaurants & bar
-Golf
-Bureau
-Gemstone Jewellery
-Comics
-Medizin & Labor
-Zubeh?r
-Exercise & Fitness
-Golf Memorabilia
-Reise & Regionales
-Akcesoria dla artyst?w
-DVDs & Blu-rays
-Kleidung, Schuhe & Accessoires
-Baby Gear
-Frankrijk
-Kochen & Genie?en
-Zubeh?r
-Pi?ces D?tach?es
-Handys ohne Vertrag
-Kids' Clothing, Shoes & Accs
-Glass & Mosaics
-Rugby Union
-Cardmaking & Scrapbooking
-Sciences, culture
-Heimnetzwerk & Zubeh?r
-Percussion
-Kids' Clothing, Shoes & Accs
-Theater Memorabilia
-T?l?phones mobiles
-Skin Care
-Fragrances
-Gry
-Nose Pins
-Animals
-Firmennetzwerk, Server
-Fabriqu?s ? la main & finis
-Standmodelle
-Handys mit Vertrag
-Volkskunst
-Cultures & ?thnicit?s
-Advertising
-Handys ohne Vertrag
-Technik & Ger?te
-Babykleding - meisjes
-Fabric
-T?l?phones ? domicile
-Papiergeld Deutschland
-Home Arts & Crafts
-Laufwerke & Speichermedien
-????
-Revues, Manuels & Catalogues
-Chasse
-Radio & Lecteurs CD Portables
-Personnages de t?l? & de films
-Malerei
-Gewerbe
-Pi?ces & accessoires
-Music Memorabilia
-Baugewerbe
-Scalextric & Slot Car
-Decorative Ornaments/ Plates
-Auto-Ersatz- & -Reparaturteile
-Appliances
-Cars
-Telefony kom?rkowe
-Herrenmode
-Sonstige
-SUVs
-Construction Toys & Kits
-????
-Kids' Clothing, Shoes & Accs
-Modelleisenbahn Spur H0
-Clocks
-Motorcycle Parts, Accessories
-Militaria
-W?rzen & verfeinern
-BD
-Laufwerke & Speichermedien
-Auto-Tuning & -Styling
-Cycling
-Disneyana
-Records
-Fashion Jewelry
-Echtschmuck
-Auto-Tuning & -Styling
-Fragrances
-Auto-Ersatz- & -Reparaturteile
-Kitchen & Home
-Flakons & Seifen
-Stromschutz & Stromverteilung
-Dispositivi archiviazione dati
-Religion & Spirituality
-????/??
-Women's Clothing
-Men's Clothing
-Test Auctions
-Metallobjekte
-Wedding & Formal Occasion
-Dekoration
-Handys mit Vertrag
-Computer Components & Parts
-Auto-Motorsport
-CD
-Budownictwo
-B?rotechnik
-Kurzreisen
-Kurzreisen
-Meubles de pouponni?re
-FMCG
-Asian Antiques
-Software
-Objets du XX?me, r?cents
-Cars & Trucks
-Computer Components & Parts
-Sport
-Speelautomaten
-Radio Equipment
-Mat?riel ?lectrique & d'essai
-K?rperpflege
-Fernseher
-Teich- & Bachlaufsysteme
-Womens' Handbags & Bags
-Echtschmuck
-Vinyl
-Comics
-PC- & Videospiele
-Gadgets
-Pet Supplies
-Kochen & Genie?en
-Women's Clothing
-Performance & DJ Equipment
-Sonstige
-Literie
-Handys ohne Vertrag
-Automobile
-Kaffee & Espressomaschinen
-Cz??ci samochodowe
-iPod- & MP3-Player-Zubeh?r
-Heim-Audio & HiFi
-Camping & Outdoor
-Handy- & PDA-Zubeh?r
-Skisport & Snowboarding
-France
-Bears
-Marcofilia
-Reise & Regionales
-Augenoptik
-Sauna & Schwimmbad
-Pottery & China
-Pottery, Porcelain
-Car Parts
-Fan Apparel & Souvenirs
-Handy- & PDA-Zubeh?r
-Mobiliar & Interieur
-Bad & K?che
-Kitchen, Dining, Bar
-DVDs & Blu-ray Discs
-Worldwide
-Skisport & Snowboarding
-BD
-Heavy Equipment Parts & Accs
-Cellulari e smartphone
-Modelleisenbahn
-Sports
-Fashion & Imitation Jewellery
-Outils
-Souvenirs musicaux
-Galanteria damska i dodatki
-Cars
-Body Piercing & Jewellery
-Elektronik & Elektrotechnik
-Cars
-Cables & Connectors
-Handys ohne Vertrag
-Sachb?cher & Ratgeber
-Veranstaltungs- & DJ-Equipment
-Tobacciana
-Auto-Hi-Fi & Navigation
-Men's Accessories
-Kids' Clothing, Shoes & Accs
-S?curit?, Domotique
-Automobile
-Computer-Komponenten & -Teile
-Celebrations & Occasions
-Herren-Accessoires
-Skisport & Snowboarding
-????
-Kochen & Genie?en
-Art, Architecture & Design
-Holzbearbeitung & Tischlerei
-Sporting Goods
-Children & Young Adults
-Silver
-Housekeeping & Organization
-B?rom?bel
-?vrigt
-Fashion Jewelry
-Speakers, Webcams & Multimedia
-Auto Epoca e Moderne
-Gastro & Nahrungsmittelgewerbe
-Healthcare, Lab & Life Science
-Beads
-Tilicartes
-MP3 Players
-Comic Books
-Parfum
-Kamera- & Fotozubeh?r
-????
-Fan Apparel & Souvenirs
-Laptop & Desktop Accessories
-Armband- & Taschenuhren
-Trading Card Games
-Mobile Phones
-Men's Jewellery
-Cards
-Parfums
-Automobile
-Verre & mosa?ques
-Weed/ Pest Control
-Jeux vid?o
-Handys ohne Vertrag
-Cz??ci samochodowe
-Srebra
-Parts & Accessories
-Hunde
-Nonfiction
-Dekoration
-TV, Video & Home Audio
-Maquillage/cosm?tiques
-V?tements pour enfants
-Fashion Jewellery
-Golf
-DVDs & Blu-rays
-Light Equipment & Tools
-Cell Phone Accessories
-Fernseher
-MRO & Industrial Supply
-Mobile Phones
-Urmakeriverktyg
-Yard, Garden & Outdoor Living
-Bijoux d'?poque, anciens
-Women's Accessories
-Test Auctions
-Vehicle Electronics & GPS
-Studium & Wissen
-Vintage Clothing & Accessories
-Cookware, Dining & Bar
-Men's Accessories
-Reit- & Fahrsport
-Women's Accessories
-Motorradkleidung
-Muziek
-Szk?o i kryszta?y
-Makeup
-Vinyl
-Serien & Lizenzprodukte
-Golf
-Fiction
-Vintage & Antique Jewellery
-Kleidung, Schuhe & Accessoires
-Reit- & Fahrsport
-Handy- & PDA-Zubeh?r
-Agrar, Forst & Kommune
-Computer Components & Parts
-Golf
-Mobile Phones
-Art. do przechowywania
-Travaux de r?novation
-TV- & Heim-Audio-Zubeh?r
-Weitere Uhren
-S??es & Salziges
-Gardening
-Plants/ Seeds/ Bulbs
-Loose Gemstones
-Fashion Jewellery
-Werkzeug
-Engagement & Wedding
-Kurzreisen
-Lots en gros
-Water Sports
-Paper Money: World
-Drives, Storage & Blank Media
-PC-Spiele
-Furniture
-DVDs & Blu-ray Discs
-Heim-Audio & HiFi
-Porzellan & Keramik
-Light Equipment & Tools
-Fiction
-Damenschuhe
-Echtschmuck
-B?rom?bel
-St?mplar
-Herrenschuhe
-Cultures & Ethnicities
-Damenmode
-Armband- & Taschenuhren
-Dekoration
-Handy- & PDA-Zubeh?r
-Parfum, eau en after shave
-Kochen & Genie?en
-G?n?alogie
-Internationale Antiq. & Kunst
-Pubblicitario
-??/??
-Fanartikel & Merchandise
-Fishing
-Stripboeken
-Test Auctions
-Other Crafts
-Kabel & Steckverbinder
-Cables & Connectors
-Golf
-Soins du corps
-Music Memorabilia
-????
-AGD
-Gitarren
-Cars
-Medizin & Labor
-Bath
-Werkzeuge & Werkstattbedarf
-Tools
-?????????
-Makeup
-Souvenirs & Travel Memorabilia
-Clothing, Helmets & Protection
-Glass
-Sports de Balle & Ballon
-Fabriqu?s ? la main & finis
-Produktions- & Industriebedarf
-Vintage
-Computer-Komponenten & -Teile
-Gastro & Nahrungsmittelgewerbe
-Software
-Souvenirs historiques
-Cucina
-Test Auctions
-Metallbearbeitung & Schlosser.
-?????
-Plants/ Seeds/ Bulbs
-VHS-video?s
-Anlageobjekte/Versteigerungen
-V?tements pour enfants
-Vinyl
-Stampanti, scanner e forniture
-Notebook- & Desktop-Zubeh?r
-Apple iPod Accessories
-Celebrations & Occasions
-Produktions- & Industriebedarf
-Kindermode, Schuhe & Access.
-Porcellana e Ceramica
-Makeup
-Obr?bka metali i ?lusarstwo
-Reklame & Werbung
-Action Figures
-Papier, B?ro- & Schreibwaren
-Wholesale, Bulk Lots
-Kleidung, Schuhe & Accessoires
-Kindermode, Schuhe & Access.
-Damenmode
-Echtschmuck
-Bedding
-V?tements & souvenirs de fan
-Weitere Wintersportarten
-Papier, B?ro- & Schreibwaren
-Modeschmuck
-Notebook- & Desktop-Zubeh?r
-Druckerei & Copyshop
-Teich- & Bachlaufsysteme
-Fragrances
-Teenskleidung M?dchen
-Coins: US
-Odzie? m?ska
-Antiquarische B?cher
-Baugewerbe
-Herrenmode
-Kindermode, Schuhe & Access.
-Thematics
-Skisport & Snowboarding
-Automobilia
-Perles, Fabrication de bijoux
-Sports de plein air
-Men's Shoes
-Wedding Supplies
-Bureau
-Handys ohne Vertrag
-Cellulari e smartphone
-Weitere Sammelgebiete
-Elektronika i elektrotechnika
-Motive
-Digital Cameras
-??????
-Clothing, Merchandise, Media
-Other Baby & Mom
-Reit- & Fahrsport
-Tragbare Disc-Player & Radios
-Modelleisenbahn Spur H0
-Card Games
-Golf
-Wholesale Lots
-?????
-Zubeh?r
-Voetbal
-Scooters
-Glass
-Music Memorabilia
-Animals
-Central & South America
-Cables & Connectors
-Papier, B?ro- & Schreibwaren
-Reit- & Fahrsport
-Software
-Camera & Photo Accessories
-Fashion Jewelry
-Kids' Clothes, Shoes & Accs.
-Herren-Accessoires
-Loose Diamonds & Gemstones
-Technik & Ger?te
-B?b?-Gar?on
-Bi?uteria z pere?
-Medizin & Labor
-K?nstlerbedarf
-??
-Car, Truck Parts
-Baby & Toddler Clothing
-Parts & Accessories
-Cars & Trucks
-Nail Care, Manicure & Pedicure
-Kindermode, Schuhe & Access.
-Notebook- & Desktop-Zubeh?r
-Heim-Audio & HiFi
-63517
-Komputery i cz??ci Apple
-Fische & Aquarien
-Englischsprachige B?cher
-Reise & Regionales
-Cz??ci samochodowe
-Kaffee & Espressomaschinen
-Heavy Equipment
-DVDs & Blu-rays
-Romanhefte
-Regional Movies
-Radsport
-Damenmode
-B?rotechnik
-Spezielle Fahrzeug-Teile
-Modelarstwo
-Boat Parts, Accessories
-Bath
-Feste & Besondere Anl?sse
-DVDs & Blu-rays
-Car Wheels, Tyres & Trims
-Hard Disk Drives
-Tuinieren en klussen
-Kleidung, Schuhe & Accessoires
-Auto-Anbau- & -Zubeh?rteile
-Pet Memorials & Urns
-C?bles & connecteurs
-Men's Clothing
-Militaria
-Vintage
-Hair Removal & Shaving
-Software
-Middle East
-Drucker, Scanner & Zubeh?r
-Geschenk- & Werbeartikel
-Fan Apparel & Souvenirs
-Armband- & Taschenuhren
-Hunde
-Dekoracje
-Cables & Connectors
-Plakate & Kunstdrucke
-Guitar
-Relojes de Pulsera
-Clothing, Helmets & Protection
-Audio
-Armband- & Taschenuhren
-Internationale Antiq. & Kunst
-PC- & Videospiele
-Heimnetzwerk & Zubeh?r
-Kleidung, Schuhe & Accessoires
-Herren-Designerbekleidung
-??????/??
-Winter Sports
-Fische & Aquarien
-Equipos de DJ y espect?culos
-Cookware, Dining & Bar
-Clothing, Merchandise, Media
-Jewel Watches
-BD
-Turismos
-Spielzeug
-Damenmode
-Livres de bandes dessin?es
-Trading Cards
-iPods & MP3-Player
-Antiquarische B?cher
-Jouet pr?scolaire, jeu de r?le
-Health Care
-Non-Fiction
-Motorrad- & Kraftradteile
-UK (Great Britain)
-Postcards
-Kleidung, Schuhe & Accessoires
-Zegarki designerskie na r?k?
-Fashion Jewelry
-Vinyles 33T
-Auto-Ersatz- & -Reparaturteile
-Jeux vid?o
-Damenmode
-Unknown
-Radio Communication
-Action Figures
-Deutschland
-Chitarre e Bassi
-Damenmode
-DVDs & Blu-rays
-Cz??ci samochodowe: Tuning
-Computer-Komponenten & -Teile
-Handy- & PDA-Zubeh?r
-F?ves
-Acciones y Papeles
-Comics
-Damen-Accessoires
-Chauffage & Climatisation
-Fragrances
-Armband- & Taschenuhren
-Damenmode
-Lettres, Vieux papiers
-Pokemon
-V?tements & souvenirs de fan
-Mobile Phones
-Jongenskleding
-Medizin & Labor
-Home Decor
-Mobile Phone & PDA Accessories
-DVDs & Blu-rays
-Damenmode
-Drives, Storage & Media
-Kinderwagen & -tragen
-??
-D?coration de maison
-Serier USA
-??????/??
-Home Decor
-Trading Card Games
-Stamps
-Puppen
-B?ro, Papier & Schreiben
-Action Figures
-Fishing
-Damenmode
-Zubeh?r & Hardware
-Disneyana
-Nonfiction
-????
-Bijoux pour hommes
-Video Equipment
-Men's Clothing
-Motorrad- & Kraftradteile
-Damenmode
-Electrical & Test Equipment
-Gewerbe
-Dyski i nap?dy
-Herrenmode
-Versandmaterialien
-Analoge Fotografie
-Camping & Outdoor
-Consommables
-Film & Fernsehen
-Make-up
-Motorrad- & Kraftradteile
-??
-Accessori foto e videocamere
-Zubeh?r
-Team Sports
-Diecast & Vehicles
-Mobile Accessories
-Reit- & Fahrsport
-Europa
-Militaria
-Bijouterie de qualit?
-Studium & Wissen
-CD Single & Maxi
-Laufwerke & Speichermedien
-V?tements pour enfants
-Handys ohne Vertrag
-Portable Audio & Headphones
-Holiday & Seasonal
-Fische & Aquarien
-???
-Women's Shoes
-Pinbacks, Bobbles, Lunchboxes
-Motorrad- & Kraftradteile
-Sports d'hiver
-Fan Shop
-V?tements pour enfants
-PC- & Videospiele
-Audio Books
-Test Auctions
-Scrapbooking & Paper Crafts
-Games
-Groothandel & Restpartijen
-Parts & Accessories
-Weitere Formate
-Cartes de jeux de collection
-Damenmode
-Firmennetzwerk, Server
-Mobile Phones
-Design & Stil
-Perles au d?tail
-Unknown
-Silver
-Cycling
-Beleuchtung
-Locaux: mat?riel, fournitures
-Uomo: Accessori
-Paper Money: World
-Edelsteine
-Science-Fiction
-Publicit?
-Tennis & sports de raquette
-Sicherheitstechnik
-Kinderkleding- jongens
-Kleidung, Schuhe & Accessoires
-Books & Manuscripts
-Autographs-Original
-Anlageobjekte/Versteigerungen
-Kunst & Kultur
-Meters, Testers, Probes
-Soins de sant?, sciences vie
-Haushaltsbatterien & Strom
-Embroidery
-Adult Only
-Hunde
-Damenmode
-Software
-Classic Toys
-Mobiliar & Interieur
-Motorrad- & Kraftradteile
-Auto's
-Hobby & K?nstlerbedarf
-Auto-Ersatz- & -Reparaturteile
-Wargames & Role-Playing
-Damenmode
-Damenmode
-Papier, B?ro- & Schreibwaren
-Vehicle Electronics & GPS
-Accessoires Image, Son
-Damenmode
-Computing
-Cassettes vid?o
-B?b?-Fille
-Makeup
-Edelsteine
-Holzspielzeug
-Dolls & Toys
-Automobiles & camions
-Hi-Fi, son, mat?riel audio
-Damenschuhe
-TV, Video & Home Audio
-V?tements b?b?s, tout-petits
-Echtschmuck
-Militaria
-Figurines d'action
-Hand Tools & Equipment
-Fu?ball-Fanshop
-Women's Clothing
-Pi?ces & accessoires
-Movie Memorabilia
-Art du Monde
-Needlecrafts & Yarn
-Vinyles
-V?tements & bijoux hors s?rie
-Obr?bka metali i ?lusarstwo
-Damenmode
-Handys ohne Vertrag
-C?bles & Alimentations
-Pflanzen, B?ume & Str?ucher
-Damenmode
-Heimnetzwerk & Zubeh?r
-Kost?me & Verkleidungen
-Software
-Herrenmode
-Skin Care
-Glass
-Test ench?res
-Auto-Ersatz- & -Reparaturteile
-Chaussures pour hommes
-Aparatos y M?quinas Antiguas
-Education & Professional
-Cell Phone Accessories
-Kochen & Genie?en
-Maquillage/cosm?tiques
-Non-Fiction
-Lenses & Filters
-Pi?ces & accessoires
-Composants & pi?ces d'ordi
-Film-Fanartikel
-Cards
-Baustoffe
-Damenmode
-Lighting & Studio
-Markenbekleidung Jungen
-Orologi da Polso: alla Moda
-Europese niet-euromunten
-Notebooks & Netbooks
-Perles, Fabrication de bijoux
-Brass
-Manufacturing & Metalworking
-Diamants, gemmes non sertis
-Monitore, Projektoren & Zub.
-??????
-Computer-Komponenten & -Teile
-VHS Tapes
-Echtschmuck
-Machinerie lourde
-Werkzeuge & Werkstattbedarf
-Bears
-Grafik, Drucke
-Damenmode
-Video Games
-Jeunesse
-Health Care
-TV, Video & Home Audio
-Baugewerbe
-Fu?ball
-Laptop & Desktop Accessories
-Echtschmuck
-Women's Clothing
-Golf
-Art du XIX?me, et avant
-Echtschmuck
-Enterprise Networking, Servers
-Radsport
-Reise & Regionales
-Car Parts
-Fan Apparel & Souvenirs
-Home Networking & Connectivity
-Auto - Hi-Fi & Navigation
-Echtschmuck
-Grundst?cke
-Vintage
-Fragrances
-Diecast & Toy Vehicles
-DVDs & Blu-rays
-Romans Policiers Suspense
-H?rb?cher & H?rspiele
-?bernachtungen
-Uomo: Accessori
-Golf
-Kindermode, Schuhe & Access.
-Jeux de construction
-Olympic Memorabilia
-Gewerbe
-Wedding & Formal Occasion
-Sheet Music & Song Books
-Diaporama
-Papier, B?ro- & Schreibwaren
-Men's Jewelry
-Building Toys
-Zubeh?r
-Accessories
-Vanity/ Perfume/ Grooming
-Auto-Tuning & -Styling
-M?nzen Deutschland ab 1945
-CDs
-Parts & Accessories
-Hand Tools & Equipment
-Fine Jewellery
-V?tements/accessoires d'?poque
-Knitting
-Cables & Connectors
-Aircraft & Aviation
-Books & Manuscripts
-MRO & Industrial Supply
-Fashion Jewellery
-BD
-Software
-Vinyles 45T
-Angelsport
-Silver
-Pflanzen, B?ume & Str?ucher
-Sporting Goods
-K?nstlerbedarf
-Indian, Hindi VCDs
-Puppen
-K?nstlerbedarf
-Kleinger?te K?che
-PC- & Videospiele
-Sports
-Italia
-Postcards
-Transportation
-Kochen & Genie?en
-Sport
-Pottery & China
-Mobile Phones
-Fu?ball
-Kindermode, Schuhe & Access.
-Vintage
-Medizin & Labor
-V?tements b?b?s, tout-petits
-Computer-Komponenten & -Teile
-Knives, Swords & Blades
-Dog Supplies
-Diecast & Toy Vehicles
-Th?mes
-T?l?, vid?o & audio domestique
-Cuisine
-Tauchen
-Women's Clothing
-Women's Clothing
-Memorabilia
-Computer-Klassiker
-Nonfiction
-Haushalt
-Test Auctions
-Software
-Sculptures
-Kunststoffindustrie & Chemie
-Animation
-Mobile Phones
-Objets publicitaires
-Dolls
-Kochen & Genie?en
-Produkcja i Przemys?
-Pet Accessories
-Parts & Accessories
-Dolls
-Herrenmode
-Fragrances
-Aktionen
-F?tes & saisonniers
-Europa
-Herrenmode
-LCD / TFT Monitors
-???
-Home Networking & Connectivity
-Drives, Storage & Blank Media
-Software
-Teleskope & Ferngl?ser
-Fashion Jewelry
-Agriculture
-M?nzen Deutschland ab 1945
-Handy- & PDA-Zubeh?r
-Apple iPods
-Advertising
-Watches
-Scrapbooking & Paper Crafts
-??
-Monitors, Projectors & Accs.
-Decorative Collectibles
-Mobile Phones
-Land
-Damenmode
-Partitions/Livres de chansons
-RC-Modellbau
-Bi?uteria do piercingu
-Bijoux d'?poque, anciens
-Software
-?lectronique & GPS de v?hicule
-Food & Beverages
-??
-Damenmode
-Parts & Accessories
-Plakate & Kunstdrucke
-Ansichtkaarten en foto's
-Cross Stitch
-Motorrad- & Kraftradteile
-Damenmode
-Skisport & Snowboarding
-Spa, Kosmetik- & Friseursalon
-Artisanat ? l'aiguille & fil
-Glass
-Literie
-Batteries, percussions
-Fan Apparel & Souvenirs
-Glass
-Creative Services
-Sewing & Fabric
-Makeup
-Clothes, Shoes & Accessories
-Watches
-Yard, Garden & Outdoor Living
-Imprimantes, scanneurs & acc.
-Revues, manuels, catalogues
-Antiquit?s asiatiques
-Notes
-T?l?phones fixes, r?pondeurs
-Bijoux de mode
-Sega
-Clothes, Shoes & Accessories
-Electrical & Test Equipment
-Bijoux de mode
-Camcorder
-Modellautos
-Accesorios, tablets e eBooks
-Bureau
-Kultur & Events
-Europa
-Feste & Besondere Anl?sse
-Auto - Hi-Fi & Navigation
-Gartenger?te
-Fantaisie/mythe/magie
-Affiches
-Desktop PCs
-Golf
-Tasteninstrumente
-Golf
-Major Appliances
-Pi?ces Europe (pre-euro)
-Motocyclettes
-?ducatifs
-??????
-Vintage
-Cards
-Auto-Ersatz- & -Reparaturteile
-Damenmode
-Kleinger?te Pflege & Wellness
-Elettrodomestici
-D?coration de maison
-Historical Memorabilia
-Herrenschuhe
-Action Figures
-Druckerei & Copyshop
-Hunting
-Automobilia
-Baby Clothing
-Fabric
-LEGO
-Decorative Collectibles
-M?taphysique & Nouvel ?ge
-Audio - Composants & Syst?mes
-Herrenschuhe
-Antikspielzeug
-Automobile
-Reit- & Fahrsport
-??
-Vini rossi: Toscana
-Cycling
-Coins: World
-Cartes de jeux de collection
-Cz??ci samochodowe
-??/??
-Bijoux de mode
-Adult Unisex
-Elektrowerkzeuge
-Publicit?
-Computer Components & Parts
-Verre, cristal
-Souvenirs & Travel Memorabilia
-A/V Accessories & Cables
-Varia
-Internationale Antiq. & Kunst
-Laptops & Netbooks
-Glas?gon
-World Coins
-Nintendo Gamecube
-Models & Kits
-Pottery & China
-Handys ohne Vertrag
-Textbooks, Education
-????/??
-Radsport
-Bijoux de mode
-Echtschmuck
-Computer-Komponenten & -Teile
-Clothes, Shoes & Accessories
-Golf
-Services & Real Estate
-Damenmode
-Sachb?cher & Ratgeber
-DVD, Blu-ray & Heimkino
-Bew?ssern
-Staubsauger
-Kinderkleding- meisjes
-Vintage
-M?bel
-Porzellan & Keramik
-Fu?ball
-Dispositivi archiviazione dati
-Linens & Textiles (1930-Now)
-Comics
-???
-Accessories
-Scrapbooking
-Staubsauger
-Sauna & Schwimmbad
-Middle East
-Bijoux de mode
-Computer-Komponenten & -Teile
-Portable Audio & Headphones
-Agrar, Forst & Kommune
-Disneyana
-Software
-Soins de sant?, sciences vie
-Electronic Pets
-Linge & textile (avant 1930)
-Porzellan & Keramik
-Radsport
-V?tements pour enfants
-Budownictwo
-Fashion Jewelry
-Action Figures
-Zapatillas
-Skiing & Snowboarding
-Fotocamere digitali
-Dolls
-Medizin & Labor
-Car Accessories
-Bricolage e fai da te
-Construction
-Car Parts
-Elektrowerkzeuge
-Mobile & Smart Phones
-Outdoor Sports
-Science fiction & horreur
-Odzie? pozosta?a
-Festnetztelefone & Zubeh?r
-Echtschmuck
-CDs & DVDs
-Decorative Collectibles
-Fragrances
-Exercise & Fitness
-Loose Diamonds & Gemstones
-Hochzeit & Besondere Anl?sse
-Baby & Toddler Clothing
-Tickets, Experiences
-Wholesale Lots
-Vintage & Antique Jewellery
-??CD
-Handys ohne Vertrag
-CD Players
-Animals
-Fine Jewellery
-Wand & Boden
-Printing & Graphic Arts
-Echtschmuck
-Dvd's - Films
-Computer-Komponenten & -Teile
-Historische Wertpapiere
-Music Memorabilia
-Fitness
-Software
-Jogging
-Damesschoenen
-Enfants: v?tements, access.
-Radio Communication
-Pokemon
-Photographic Images
-Home Improvement
-Kindermode, Schuhe & Access.
-Produktions- & Industriebedarf
-Ansichtskarten
-Power Protection, Distribution
-Autoreifen & Felgen
-Team Sports
-Anlageobjekte/Versteigerungen
-Commonwealth/c. britannique
-Objets de collection de d?cora
-Trading Cards
-MP3 Player Accessories
-Automobilia
-Papiergeld Deutschland
-Glass
-iPods & MP3-Player
-Pre-School & Young Children
-Kamera- & Fotozubeh?r
-Kleidung, Schuhe & Accessoires
-Familienplanung
-Men's Jewelry
-Women's Clothing
-Otros deportes
-Poligrafia
-Baby Clothing
-Fische & Aquarien
-Heavy Equipment Attachments
-Software
-????
-Nakrycia sto?owe
-Bricolage: Outils
-Software
-Fashion Jewelry
-Home Arts & Crafts
-Damenmode
-Damenmode
-Audio- & Video-Rarit?ten
-Souvenirs & Travel Memorabilia
-Transport
-Brett- & Gesellschaftsspiele
-Guitar
-Animals
-Enterprise Networking, Servers
-Damenschuhe
-Cz??ci samochodowe
-Pneumatici, Cerchi e Ruote
-Women's Wrist Watches
-Alkoholfreie Getr?nke
-Mobile Phones
-Vinos y Gastronom?a
-Sports de plein air
-Hobby & K?nstlerbedarf
-Handy- & PDA-Zubeh?r
-Calzado mujer
-Personnages de t?l? & de films
-????
-Weitere Ballsportarten
-Other Video Games & Consoles
-Garden & Patio Furniture
-Portable Audio & Headphones
-Jeux
-Journaux, revues, magazines
-Parts & Accessories
-Fu?ball-Fanshop
-Diecast & Toy Vehicles
-Colliers
-Women's Shoes & Sandals
-Computer-Komponenten & -Teile
-Sciences, culture
-Camping & Outdoor
-Stamping & Embossing
-Car Parts
-Kleinkindspielzeug
-Gro?handel & Sonderposten
-Dekoration
-Kinderm?bel & Wohnen
-Cromos
-Fitness
-Mobility, Disability & Medical
-Remorques
-Kurzreisen
-Reise & Regionales
-Papier, B?ro- & Schreibwaren
-Men's Jewellery
-VHS Tapes
-Golf
-Textbooks
-Malerei
-Yard, Garden & Outdoor Living
-Mobile Phone & PDA Accessories
-Mobile Phone Accessories
-%F%l%S%2!<%`
-Feste & Besondere Anl?sse
-Bijoux de mode
-Skisport & Snowboarding
-Golf
-Teich- & Bachlaufsysteme
-Fashion Jewellery
-Animals
-Printers, Scanners & Supplies
-Auto-Tuning & -Styling
-Kids' Clothes, Shoes & Accs.
-Tuning & Styling
-Spielzeug
-Bistrot
-Kochen & Genie?en
-Antiquarische B?cher
-Camcorders
-Skisport & Snowboarding
-Literatur & Kochb?cher
-Fragrances
-Schulbedarf
-M?bel
-Passenger Vehicles
-Auto-Ersatz- & -Reparaturteile
-Tools
-Spezielle Fahrzeug-Teile
-Kindermode, Schuhe & Access.
-Kleidung, Schuhe & Accessoires
-Hunting
-PC- & Videospiele
-Watches
-Nachschlagewerke
-Decorative Ornaments/ Plates
-Piriphiriques PC
-Music Memorabilia
-Records
-Computer & Internet
-Adult Only
-Marine/ Maritime
-Machinerie lourde
-Mobiele Telefoon Accessoires
-Memory Cards & Readers
-Militaria
-Weine
-Paper Money: World
-Pen Drives
-Kinderwagen & -tragen
-Spielzeug f?r drau?en
-Home Decor
-Kleidung, Schuhe & Accessoires
-Maquillage/cosm?tiques
-Skisport & Snowboarding
-Clocks
-Bolsos
-Homeware, Kitchenware
-Damenmode
-Medizin & Labor
-Football Shirts
-Womens' Shoes
-Studium & Wissen
-Zestawy mebli
-Rollos, Gardinen & Vorh?nge
-Romans, litt?rature
-Militaria
-Wand & Boden
-Skisport & Snowboarding
-Festnetztelefone & Zubeh?r
-Travaux de r?novation
-Pi?ces & accessoires
-Music Memorabilia
-Haushaltsstrom
-Fan Shop
-Werkzeuge
-Europe
-V?tements & souvenirs de fan
-B?rom?bel
-Cards
-Cuisine, restaurants & bar
-Transport
-Auto-Tuning & -Styling
-Zubeh?r & Hardware
-Zubeh?r
-Thema's
-Autographs
-Sonnenschutz
-Tragbare Disc-Player & Radios
-W?osy
-Lichaam & Hygi?ne
-Sicurezza e Antifurti
-Akcesoria do konsol
-Home Improvement
-Cartes Postales
-W?rzen & verfeinern
-Xbox 360
-Laufwerke & Speichermedien
-Pet Supplies
-?bernachtungen
-Kamera- & Fotozubeh?r
-Shaving & Hair Removal
-Holiday & Seasonal
-Drives, Storage & Blank Media
-Hobby & K?nstlerbedarf
-Fietsen en fietssport
-Furniture
-Vintage Vehicles
-Handcrafted & Finished Pieces
-Television Memorabilia
-Livres Toutes Disciplines FR
-Software
-Femmes: Chaussures
-DVDs & Blu-ray Discs
-Personenauto's
-Piercing-/K?rperschmuck
-Pen Drives
-Pflanzen, B?ume & Str?ucher
-Dekoration
-Baugewerbe
-Festivals & Konzerte
-Kochen & Genie?en
-Car Parts
-Coins
-Folkloreschmuck
-Fische & Aquarien
-Fashion Jewellery
-Cars & Trucks
-Damenmode
-Cycling
-Hobby & K?nstlerbedarf
-Schulbedarf
-Handys mit Vertrag
-Enthaarung & Rasur
-Souvenirs historiques
-Werkzeuge
-Motor Sport Memorabilia
-Wyposa?enie
-Computer-Komponenten & -Teile
-Celebrations & Occasions
-Souvenirs musicaux
-Cultures & Ethnicities
-Herrenschuhe
-Outremer
-Nakrycia sto?owe
-Bi?uteria do piercingu
-Animals
-Skisport & Snowboarding
-Noten & Songbooks
-Kabel & Steckverbinder
-Gesichtspflege
-Mobile Phone & PDA Accessories
-Zubeh?r
-Motorcycles
-Weitere Sportarten
-Water Sports
-Team Sports
-Architectural Antiques
-Reise & Regionales
-Indian, Hindi DVDs
-Damenmode
-Reit- & Fahrsport
-Parts & Accessories
-Kleidung, Schuhe & Accessoires
-Sciences, culture
-Pflege
-Damenmode
-S?mereien & Zwiebeln
-Sport d'acqua
-Kids & Teens at Home
-Dessins anim?s & personnages
-Fashion Jewellery
-Golf
-Conexi?n de redes
-Auto-Anbau- & -Zubeh?rteile
-Echtschmuck
-Manuali, Corsi, Libri di testo
-Srebra
-Eisenwaren
-Auto-Ersatz- & -Reparaturteile
-V?tements & souvenirs de fan
-Vanity, Perfume & Shaving
-Fashion Jewellery
-Cz??ci samochodowe
-Art. do przechowywania
-Instruments
-Rocks, Fossils & Minerals
-Geschenk- & Werbeartikel
-??????????
-Akcesoria kuchenne
-Men's Clothing
-Adult Only
-Tools
-Modellautos
-Playmobil
-Manik?re & Pedik?re
-Fashion Jewelry
-Sewing (1930-Now)
-Fu?ball-Fanshop
-Weitere Uhren
-Fantasy, Magic
-Football
-Elektronik & Elektrotechnik
-Makeup
-Fan Apparel & Souvenirs
-Caravanas y Remolques
-Paper
-Parts & Accessories
-Professional, Medical, Dental
-Heim-Audio & HiFi
-TV-Receiver & Set-Top-Boxen
-Souvenirs militaires
-Painting, Drawing & Art
-Trains/ Railway Models
-Holzbearbeitung & Tischlerei
-Jeux de construction
-M?bel
-Staubsauger
-Scrapbooking & Paper Crafts
-CDs
-????
-Fan Apparel & Souvenirs
-Mariage & tenue habill?e
-Bath
-Analoge Fotografie
-Argento
-B?rotechnik
-Costume Jewellery
-Armband- & Taschenuhren
-Fernseher
-Pi?ces - ?chelle mondiale
-Obr?bka metali i ?lusarstwo
-Automobile
-Building Materials, DIY
-Meble
-Cross Stitch
-Barware
-Spezielle Fahrzeug-Teile
-Damenmode
-Cars
-Militaria
-Men's Clothing
-Accessori cellulari e palmari
-Collectibles & Memorabilia
-Cartes de collection
-Home Improvement
-Construction Toys & Kits
-Donna: Scarpe
-Enterprise Networking, Servers
-Radsport
-Armband- & Taschenuhren
-F?tes & saisonniers
-Vinyles 33T
-Ricambi Moto d'Epoca
-Dame
<TRUNCATED>
[16/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
deleted file mode 100644
index f10712a..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.net.util.Base64;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IIDesc extends RootPersistentEntity {
-
- public static final String HBASE_FAMILY = "f";
- public static final String HBASE_QUALIFIER = "c";
- public static final byte[] HBASE_FAMILY_BYTES = Bytes.toBytes(HBASE_FAMILY);
- public static final byte[] HBASE_QUALIFIER_BYTES = Bytes.toBytes(HBASE_QUALIFIER);
-
- private KylinConfig config;
- private DataModelDesc model;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("model_name")
- private String modelName;
- @JsonProperty("timestamp_dimension")
- private String timestampDimension;
- @JsonProperty("value_dimensions")
- private List<IIDimension> valueDimensions;
- @JsonProperty("metrics")
- private String[] metricNames;
- @JsonProperty("sharding")
- private short sharding = 1; // parallelism
- @JsonProperty("slice_size")
- private int sliceSize = 50000; // no. rows
- @JsonProperty("signature")
- private String signature;
-
- // computed
- private List<TableDesc> allTables = Lists.newArrayList();
- private List<TblColRef> allColumns = Lists.newArrayList();
- private List<TblColRef> allDimensions = Lists.newArrayList();
- private int tsCol;
- private int[] valueCols;
- private int[] metricsCols;
- private BitSet metricsColSet;
- private List<MeasureDesc> measureDescs;
-
- public void init(MetadataManager metadataManager) {
-
- config = metadataManager.getConfig();
-
- if (this.modelName == null || this.modelName.length() == 0) {
- throw new RuntimeException("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
- }
-
- this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
-
- if (this.model == null) {
- throw new RuntimeException("No data model found with name '" + modelName + "'.");
- }
-
- timestampDimension = timestampDimension.toUpperCase();
-
- // capitalize
- IIDimension.capicalizeStrings(valueDimensions);
- StringUtil.toUpperCaseArray(metricNames, metricNames);
-
- // retrieve all columns and all tables, and make available measure to ii
- HashSet<String> allTableNames = Sets.newHashSet();
- measureDescs = Lists.newArrayList();
- measureDescs.add(makeCountMeasure());
- for (IIDimension iiDimension : valueDimensions) {
- TableDesc tableDesc = this.getTableDesc(iiDimension.getTable());
- for (String column : iiDimension.getColumns()) {
- ColumnDesc columnDesc = tableDesc.findColumnByName(column);
- TblColRef tcr = new TblColRef(columnDesc);
- allColumns.add(tcr);
- allDimensions.add(tcr);
- measureDescs.add(makeHLLMeasure(columnDesc, "hllc10"));
- }
-
- if (!allTableNames.contains(tableDesc.getIdentity())) {
- allTableNames.add(tableDesc.getIdentity());
- allTables.add(tableDesc);
- }
- }
- for (String column : metricNames) {
- TableDesc tableDesc = this.getTableDesc(this.getFactTableName());
- ColumnDesc columnDesc = tableDesc.findColumnByName(column);
- allColumns.add(new TblColRef(columnDesc));
- measureDescs.add(makeNormalMeasure("SUM", columnDesc));
- measureDescs.add(makeNormalMeasure("MIN", columnDesc));
- measureDescs.add(makeNormalMeasure("MAX", columnDesc));
- if (!allTableNames.contains(tableDesc.getIdentity())) {
- allTableNames.add(tableDesc.getIdentity());
- allTables.add(tableDesc);
- }
- }
-
- // indexing for each type of columns
- valueCols = new int[IIDimension.getColumnCount(valueDimensions)];
- metricsCols = new int[metricNames.length];
- metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
-
- int totalIndex = 0;
- for (int i = 0; i < valueCols.length; ++i, ++totalIndex) {
- valueCols[i] = totalIndex;
- }
- for (int i = 0; i < metricsCols.length; ++i, ++totalIndex) {
- metricsCols[i] = totalIndex;
- metricsColSet.set(totalIndex);
- }
-
- // partitioning column
- tsCol = -1;
- for (int i = 0; i < allColumns.size(); ++i) {
- TblColRef col = allColumns.get(i);
-
- if (col.isSameAs(this.getFactTableName(), this.timestampDimension)) {
- tsCol = i;
- break;
- }
- }
- if (tsCol < 0)
- throw new RuntimeException("timestamp_dimension is not in valueDimensions");
- }
-
- private TableDesc getTableDesc(String tableName) {
- return MetadataManager.getInstance(this.config).getTableDesc(tableName);
- }
-
- public String getResourcePath() {
- return getIIDescResourcePath(name);
- }
-
- public static String getIIDescResourcePath(String descName) {
- return ResourceStore.II_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
- }
-
- public List<MeasureDesc> getMeasures() {
- return measureDescs;
- }
-
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measureDescs) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
- private MeasureDesc makeNormalMeasure(String func, ColumnDesc columnDesc) {
- String columnName = columnDesc.getName();
- String returnType = columnDesc.getTypeName();
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression(func);
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue(columnName);
- p1.setColRefs(ImmutableList.of(new TblColRef(columnDesc)));
- f1.setParameter(p1);
- f1.setReturnType(returnType);
- if (f1.isSum() && f1.getReturnDataType().isIntegerFamily()) {
- f1.setReturnType("bigint");
- }
-
- measureDesc.setFunction(f1);
- return measureDesc;
- }
-
- /**
- *
- * @param hllType represents the presision
- */
- private MeasureDesc makeHLLMeasure(ColumnDesc columnDesc, String hllType) {
- String columnName = columnDesc.getName();
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("COUNT_DISTINCT");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue(columnName);
- p1.setColRefs(ImmutableList.of(new TblColRef(columnDesc)));
- f1.setParameter(p1);
- f1.setReturnType(hllType);
- measureDesc.setFunction(f1);
- return measureDesc;
- }
-
- private MeasureDesc makeCountMeasure() {
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("COUNT");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("constant");
- p1.setValue("1");
- f1.setParameter(p1);
- f1.setReturnType("bigint");
- measureDesc.setFunction(f1);
- return measureDesc;
- }
-
- /**
- * at first stage the only table in II is fact table, tables
- *
- * @return
- */
- public List<TableDesc> listTables() {
- return allTables;
- }
-
- public List<TblColRef> listAllColumns() {
- return allColumns;
- }
-
- public List<TblColRef> listAllDimensions() {
- return allDimensions;
- }
-
- public TblColRef findColumnRef(String table, String column) {
- ColumnDesc columnDesc = this.getTableDesc(table).findColumnByName(column);
- return new TblColRef(columnDesc);
- }
-
- public int findColumn(TblColRef col) {
- return this.allColumns.indexOf(col);
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public String getName() {
- return name;
- }
-
- public String getModelName() {
- return modelName;
- }
-
- public void setModelName(String modelName) {
- this.modelName = modelName;
- }
-
- public DataModelDesc getModel() {
- return model;
- }
-
- public void setModel(DataModelDesc model) {
- this.model = model;
- }
-
- public int getTimestampColumn() {
- return tsCol;
- }
-
- public int[] getValueColumns() {
- return valueCols;
- }
-
- public int[] getMetricsColumns() {
- return metricsCols;
- }
-
- public short getSharding() {
- return sharding;
- }
-
- public int getSliceSize() {
- return sliceSize;
- }
-
- public String getSignature() {
- return signature;
- }
-
- public void setSignature(String signature) {
- this.signature = signature;
- }
-
- public boolean isMetricsCol(TblColRef col) {
- if (!col.getTable().equalsIgnoreCase(this.getFactTableName()))
- return false;
- return isMetricsCol(this.findColumn(col));
- }
-
- public boolean isMetricsCol(int index) {
- return metricsColSet.get(index);
- }
-
- /**
- * the returned fact table name is guaranteed to be in the form of db.table
- *
- * @return
- */
- public String getFactTableName() {
- return this.model.getFactTable().toUpperCase();
- }
-
- public String getTimestampDimension() {
- return timestampDimension;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String calculateSignature() {
- MessageDigest md = null;
- try {
- md = MessageDigest.getInstance("MD5");
- StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTableName()).append("|").append(timestampDimension).append("|").append("|").append(JsonUtil.writeValueAsString(valueDimensions)).append("|").append(JsonUtil.writeValueAsString(this.metricNames)).append("|").append(sharding).append("|").append(sliceSize);
-
- byte[] signature = md.digest(sigString.toString().getBytes());
- return new String(Base64.encodeBase64(signature));
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("Failed to calculate signature");
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to calculate signature");
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
deleted file mode 100644
index 56e3692..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDimension.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.List;
-
-import org.apache.kylin.common.util.StringUtil;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/26/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class IIDimension {
- @JsonProperty("table")
- private String table;
- @JsonProperty("columns")
- private String[] columns;
-
- public String getTable() {
- return table;
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public String[] getColumns() {
- return columns;
- }
-
- public void setColumns(String[] columns) {
- this.columns = columns;
- }
-
- public static void capicalizeStrings(List<IIDimension> dimensions) {
- if (dimensions != null) {
- for (IIDimension iiDimension : dimensions) {
- iiDimension.setTable(iiDimension.getTable().toUpperCase());
- StringUtil.toUpperCaseArray(iiDimension.getColumns(), iiDimension.getColumns());
- }
- }
- }
-
- public static int getColumnCount(List<IIDimension> iiDimensions) {
- int count = 0;
- for (IIDimension iiDimension : iiDimensions) {
- count += iiDimension.getColumns().length;
- }
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
deleted file mode 100644
index df24703..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.invertedindex.index.BitMapContainer;
-import org.apache.kylin.invertedindex.index.ColumnValueContainer;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class IIKeyValueCodec {
-
- public static final int SHARD_LEN = 2;
- public static final int TIMEPART_LEN = 8;
- public static final int COLNO_LEN = 2;
-
- private TableRecordInfoDigest infoDigest;
-
- public IIKeyValueCodec(TableRecordInfoDigest digest) {
- this.infoDigest = digest;
- }
-
- public Collection<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKeyValue(Slice slice) {
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result = Lists.newArrayList();
- ColumnValueContainer[] containers = slice.getColumnValueContainers();
- for (int col = 0; col < containers.length; col++) {
- if (containers[col] instanceof BitMapContainer) {
- collectKeyValues(slice, col, (BitMapContainer) containers[col], result);
- } else if (containers[col] instanceof CompressedValueContainer) {
- collectKeyValues(slice, col, (CompressedValueContainer) containers[col], result);
- } else {
- throw new IllegalArgumentException("Unkown container class " + containers[col].getClass());
- }
- }
- return result;
- }
-
- private void collectKeyValues(Slice slice, int col, CompressedValueContainer container, //
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col, -1);
- ImmutableBytesWritable value = container.toBytes();
- result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(key, value));
- }
-
- private void collectKeyValues(Slice slice, int col, BitMapContainer container, //
- ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) {
- List<ImmutableBytesWritable> values = container.toBytes();
- for (int v = 0; v < values.size(); v++) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col, v);
- result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(key, values.get(v)));
- }
- }
-
- ImmutableBytesWritable encodeKey(short shard, long timestamp, int col, int colValue) {
- byte[] bytes = new byte[20];
- int len = encodeKey(shard, timestamp, col, colValue, bytes, 0);
- return new ImmutableBytesWritable(bytes, 0, len);
- }
-
- int encodeKey(short shard, long timestamp, int col, int colValue, byte[] buf, int offset) {
- int i = offset;
-
- BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
- i += SHARD_LEN;
- BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN);
- i += TIMEPART_LEN;
-
- BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
- i += COLNO_LEN;
-
- if (colValue >= 0) {
- int colLen = infoDigest.length(col);
- BytesUtil.writeUnsigned(colValue, buf, i, colLen);
- i += colLen;
- }
-
- return i - offset;
- }
-
- public Iterable<Slice> decodeKeyValue(Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
- return new Decoder(infoDigest, kvs);
- }
-
- private static class Decoder implements Iterable<Slice> {
-
- TableRecordInfoDigest info;
- Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator;
-
- Slice next = null;
- short curShard = Short.MIN_VALUE;
- long curSliceTimestamp = Long.MIN_VALUE;
- int curCol = -1;
- int curColValue = -1;
- short lastShard = Short.MIN_VALUE;
- long lastSliceTimestamp = Long.MIN_VALUE;
- int lastCol = -1;
- ColumnValueContainer[] containers = null;
- List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
-
- Decoder(TableRecordInfoDigest info, Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
- this.info = info;
- this.iterator = kvs.iterator();
- }
-
- private void goToNext() {
- if (next != null) { // was not fetched
- return;
- }
-
- // NOTE the input keys are ordered
- while (next == null && iterator.hasNext()) {
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> kv = iterator.next();
- ImmutableBytesWritable k = kv.getFirst();
- ImmutableBytesWritable v = kv.getSecond();
- decodeKey(k);
-
- if (curShard != lastShard || curSliceTimestamp != lastSliceTimestamp) {
- makeNext();
- }
- consumeCurrent(v);
- }
- if (next == null) {
- makeNext();
- }
- }
-
- private void decodeKey(ImmutableBytesWritable k) {
- byte[] buf = k.get();
- int i = k.getOffset();
-
- curShard = (short) BytesUtil.readUnsigned(buf, i, SHARD_LEN);
- i += SHARD_LEN;
- curSliceTimestamp = BytesUtil.readLong(buf, i, TIMEPART_LEN);
- i += TIMEPART_LEN;
-
- curCol = BytesUtil.readUnsigned(buf, i, COLNO_LEN);
- i += COLNO_LEN;
-
- if (i - k.getOffset() < k.getLength()) {
- // bitmap
- int colLen = info.length(curCol);
- curColValue = BytesUtil.readUnsigned(buf, i, colLen);
- i += colLen;
- } else {
- // value list
- curColValue = -1;
- }
- }
-
- private void consumeCurrent(ImmutableBytesWritable v) {
- if (curCol != lastCol && bitMapValues.size() > 0) { // end of a
- // bitmap
- // container
- addBitMapContainer(lastCol);
- }
- if (curColValue < 0) {
- CompressedValueContainer c = new CompressedValueContainer(info, curCol, 0);
- c.fromBytes(v);
- addContainer(curCol, c);
- } else {
- assert curColValue == bitMapValues.size();
- // make a copy, the value object from caller is typically reused
- // through iteration
- bitMapValues.add(new ImmutableBytesWritable(v));
- }
-
- lastShard = curShard;
- lastSliceTimestamp = curSliceTimestamp;
- lastCol = curCol;
- }
-
- private void makeNext() {
- if (bitMapValues.isEmpty() == false) {
- addBitMapContainer(lastCol);
- }
- if (containers != null) {
- next = new Slice(info, lastShard, lastSliceTimestamp, containers);
- }
- lastSliceTimestamp = Long.MIN_VALUE;
- lastCol = -1;
- containers = null;
- bitMapValues.clear();
- }
-
- private void addBitMapContainer(int col) {
- BitMapContainer c = new BitMapContainer(info, col);
- c.fromBytes(bitMapValues);
- addContainer(col, c);
- bitMapValues.clear();
- }
-
- private void addContainer(int col, ColumnValueContainer c) {
- if (containers == null) {
- containers = new ColumnValueContainer[info.getColumnCount()];
- }
- containers[col] = c;
- }
-
- @Override
- public Iterator<Slice> iterator() {
- return new Iterator<Slice>() {
- @Override
- public boolean hasNext() {
- goToNext();
- return next != null;
- }
-
- @Override
- public Slice next() {
- Slice result = next;
- next = null;
- return result;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
deleted file mode 100644
index bfcc7d5..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.tools;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-
-/**
- * @author yangli9
- */
-public class IICLI {
-
- public static void main(String[] args) throws IOException {
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- String iiName = args[0];
- IIInstance ii = mgr.getII(iiName);
-
- String path = args[1];
- System.out.println("Reading from " + path + " ...");
-
- TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- int count = 0;
- for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
- for (RawTableRecord rec : slice) {
- System.out.printf(new TableRecord(rec, info).toString());
- count++;
- }
- }
- System.out.println("Total " + count + " records");
- }
-
- public static Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> readSequenceKVs(Configuration hconf, String path) throws IOException {
- final Reader reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path)));
- return new Iterable<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>() {
- @Override
- public Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> iterator() {
- return new Iterator<Pair<ImmutableBytesWritable, ImmutableBytesWritable>>() {
- ImmutableBytesWritable k = new ImmutableBytesWritable();
- ImmutableBytesWritable v = new ImmutableBytesWritable();
- Pair<ImmutableBytesWritable, ImmutableBytesWritable> pair = new Pair<ImmutableBytesWritable, ImmutableBytesWritable>(k, v);
-
- @Override
- public boolean hasNext() {
- boolean hasNext = false;
- try {
- hasNext = reader.next(k, v);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- if (hasNext == false) {
- IOUtils.closeQuietly(reader);
- }
- }
- return hasNext;
- }
-
- @Override
- public Pair<ImmutableBytesWritable, ImmutableBytesWritable> next() {
- return pair;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
deleted file mode 100644
index 87723fd..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 1/30/15.
- */
-public class IIDescManagerTest extends LocalFileMetadataTestCase {
-
- public static final String TEST_II_DESC_NAME = "test_kylin_ii_desc";
-
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCRUD() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- String newDescName = "Copy_of_" + TEST_II_DESC_NAME;
-
- try {
- IIDesc testRecord = mgr.getIIDesc(newDescName);
- if (testRecord != null)
- mgr.removeIIDesc(testRecord);
- } catch (IOException e) {
- // just ensure the old one is removed
- }
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- desc.setName(newDescName);
- desc.setLastModified(0);
-
- mgr.createIIDesc(desc);
-
- desc = mgr.getIIDesc(newDescName);
-
- Assert.assertNotNull(desc);
-
- mgr.updateIIDesc(desc); // this will trigger cache wipe; please ignore the HTTP error in logs.
-
- mgr.removeIIDesc(desc);
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
-
- }
-
- @Test
- public void testReload() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- // do some modification
- desc.setUuid(UUID.randomUUID().toString());
-
- IIDesc newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertEquals(desc, newDesc);
-
- // reload the cache
- mgr.reloadIIDesc(TEST_II_DESC_NAME);
-
- newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertNotEquals(desc, newDesc);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
deleted file mode 100644
index d5e29ec..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 1/30/15.
- */
-public class IIDescTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() {
- this.createTestMetadata();
-
- }
-
- @After
- public void clear() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIDesc() {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_desc");
- DataModelDesc model = iiDesc.getModel();
- Assert.assertNotNull(iiDesc);
- Assert.assertNotNull(model);
-
- }
-
- @Test
- public void testSerialization() throws IOException {
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_desc");
- String str = JsonUtil.writeValueAsIndentString(iiDesc);
- System.out.println(str);
- @SuppressWarnings("unused")
- IIDesc desc2 = JsonUtil.readValue(str, IIDesc.class);
-
- Assert.assertEquals(iiDesc, desc2);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
deleted file mode 100644
index 50af8a4..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 2/5/15.
- */
-public class IIInstanceTest extends LocalFileMetadataTestCase {
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIsByDesc() throws IOException {
- IIManager mgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_desc");
-
- Assert.assertTrue(iiInstances.size() > 0);
-
- IIInstance instance = iiInstances.get(0);
-
- Dictionary dict = mgr.getDictionary(instance.getFirstSegment(), instance.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID"));
-
- Assert.assertNotNull(dict);
- }
-
- @Test
- public void testCreateIIInstance() throws IOException {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_desc");
-
- IIInstance ii = IIInstance.create("new_ii", "default", iiDesc);
-
- IIManager iiMgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> allIIList = iiMgr.listAllIIs();
-
- iiMgr.createII(ii);
-
- Assert.assertNotNull(iiMgr.getII("new_ii"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
deleted file mode 100644
index cfa4ba6..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.index.BitMapContainer;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.ShardingSliceBuilder;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
-
- IIInstance ii;
- TableRecordInfo info;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii");
- this.info = new TableRecordInfo(ii.getFirstSegment());
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- @Ignore
- public void testBitMapContainer() {
- // create container
- BitMapContainer container = new BitMapContainer(info.getDigest(), 0);
- Dictionary<String> dict = info.dict(0);
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container.append(v);
- }
- container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
- container.closeForChange();
-
- // copy by serialization
- List<ImmutableBytesWritable> bytes = container.toBytes();
- BitMapContainer container2 = new BitMapContainer(info.getDigest(), 0);
- container2.fromBytes(bytes);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- int value = container2.getValueIntAt(i++);
- assertEquals(v, value);
- }
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], container2.getValueIntAt(i++));
- assertEquals(container, container2);
- }
-
- @Test
- public void testCompressedValueContainer() {
- // create container
- CompressedValueContainer container = new CompressedValueContainer(info.getDigest(), 0, 500);
- Dictionary<String> dict = info.dict(0);
-
- byte[] buf = new byte[dict.getSizeOfId()];
- ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
-
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
- container.append(bytes);
- }
- BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
- container.append(bytes);
- container.closeForChange();
-
- // copy by serialization
- ImmutableBytesWritable copy = container.toBytes();
- CompressedValueContainer container2 = new CompressedValueContainer(info.getDigest(), 0, 500);
- container2.fromBytes(copy);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(v, value);
- }
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
- assertEquals(container, container2);
- }
-
- @Test
- public void testCodec() throws IOException {
- List<TableRecord> records = loadRecordsSorted();
- System.out.println(records.size() + " records");
- List<Slice> slices = buildTimeSlices(records);
- System.out.println(slices.size() + " slices");
-
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = encodeKVs(codec, slices);
- System.out.println(kvs.size() + " KV pairs");
-
- List<Slice> slicesCopy = decodeKVs(codec, kvs);
- assertEquals(slices, slicesCopy);
-
- List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
- assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
- dump(recordsCopy);
- }
-
- private List<TableRecord> loadRecordsSorted() throws IOException {
- File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
- FileInputStream in = new FileInputStream(file);
- List<String> lines = IOUtils.readLines(in, "UTF-8");
- in.close();
-
- List<TableRecord> records = Lists.newArrayList();
- for (String line : lines) {
- String[] fields = line.split(",");
- TableRecord rec = info.createTableRecord();
- for (int col = 0; col < fields.length; col++) {
- rec.setValueString(col, fields[col]);
- }
- records.add(rec);
- }
-
- Collections.sort(records, new Comparator<TableRecord>() {
- @Override
- public int compare(TableRecord a, TableRecord b) {
- long x = a.getTimestamp() - b.getTimestamp();
- if (x > 0)
- return 1;
- else if (x == 0)
- return 0;
- else
- return -1;
- }
- });
-
- return records;
- }
-
- private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
- ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
- List<Slice> slices = Lists.newArrayList();
- for (TableRecord rec : records) {
- Slice slice = builder.append(rec);
- if (slice != null)
- slices.add(slice);
- }
- List<Slice> finals = builder.close();
- slices.addAll(finals);
-
- Collections.sort(slices);
- return slices;
- }
-
- private List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
-
- List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs = Lists.newArrayList();
- for (Slice slice : slices) {
- kvs.addAll(codec.encodeKeyValue(slice));
- }
- return kvs;
- }
-
- private List<Slice> decodeKVs(IIKeyValueCodec codec, List<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> kvs) {
- List<Slice> slices = Lists.newArrayList();
- for (Slice slice : codec.decodeKeyValue(kvs)) {
- slices.add(slice);
- }
- return slices;
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- System.out.println(rec.toString());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
deleted file mode 100644
index d900f55..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.junit.Test;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
-public class LZFTest {
- @Test
- public void test() throws IOException {
-
- byte[] raw = new byte[] { 1, 2, 3, 3, 2, 23 };
- byte[] data = LZFEncoder.encode(raw);
-
- byte[] data2 = new byte[data.length * 2];
- java.lang.System.arraycopy(data, 0, data2, 0, data.length);
- ImmutableBytesWritable bytes = new ImmutableBytesWritable();
- bytes.set(data2, 0, data.length);
-
- try {
- byte[] uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/jdbc/.settings/org.eclipse.core.resources.prefs b/jdbc/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 365bbd6..0000000
--- a/jdbc/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/jdbc/.settings/org.eclipse.jdt.core.prefs b/jdbc/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a903301..0000000
--- a/jdbc/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/jdbc/.settings/org.eclipse.jdt.ui.prefs b/jdbc/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index dece0e6..0000000
--- a/jdbc/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
[29/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/docs/README.md
----------------------------------------------------------------------
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index a4c802d..0000000
--- a/docs/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Apache Kylin Documentation
-
-__All documens have been moved to Kylin website at:__
-[http://kylin.apache.org/docs](http://kylin.apache.org/docs)
-
-Please visit there for further reference.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/.gitignore
----------------------------------------------------------------------
diff --git a/examples/.gitignore b/examples/.gitignore
deleted file mode 100644
index 91fe24e..0000000
--- a/examples/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/yadesk00.remote*
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
deleted file mode 100644
index 943c0fa..0000000
--- a/examples/sample_cube/create_sample_tables.sql
+++ /dev/null
@@ -1,190 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
-
-CREATE TABLE DEFAULT.KYLIN_CAL_DT
-(
-CAL_DT date
-,YEAR_BEG_DT date
-,QTR_BEG_DT date
-,MONTH_BEG_DT date
-,WEEK_BEG_DT date
-,AGE_FOR_YEAR_ID smallint
-,AGE_FOR_QTR_ID smallint
-,AGE_FOR_MONTH_ID smallint
-,AGE_FOR_WEEK_ID smallint
-,AGE_FOR_DT_ID smallint
-,AGE_FOR_RTL_YEAR_ID smallint
-,AGE_FOR_RTL_QTR_ID smallint
-,AGE_FOR_RTL_MONTH_ID smallint
-,AGE_FOR_RTL_WEEK_ID smallint
-,AGE_FOR_CS_WEEK_ID smallint
-,DAY_OF_CAL_ID int
-,DAY_OF_YEAR_ID smallint
-,DAY_OF_QTR_ID smallint
-,DAY_OF_MONTH_ID smallint
-,DAY_OF_WEEK_ID int
-,WEEK_OF_YEAR_ID tinyint
-,WEEK_OF_CAL_ID int
-,MONTH_OF_QTR_ID tinyint
-,MONTH_OF_YEAR_ID tinyint
-,MONTH_OF_CAL_ID smallint
-,QTR_OF_YEAR_ID tinyint
-,QTR_OF_CAL_ID smallint
-,YEAR_OF_CAL_ID smallint
-,YEAR_END_DT string
-,QTR_END_DT string
-,MONTH_END_DT string
-,WEEK_END_DT string
-,CAL_DT_NAME string
-,CAL_DT_DESC string
-,CAL_DT_SHORT_NAME string
-,YTD_YN_ID tinyint
-,QTD_YN_ID tinyint
-,MTD_YN_ID tinyint
-,WTD_YN_ID tinyint
-,SEASON_BEG_DT string
-,DAY_IN_YEAR_COUNT smallint
-,DAY_IN_QTR_COUNT tinyint
-,DAY_IN_MONTH_COUNT tinyint
-,DAY_IN_WEEK_COUNT tinyint
-,RTL_YEAR_BEG_DT string
-,RTL_QTR_BEG_DT string
-,RTL_MONTH_BEG_DT string
-,RTL_WEEK_BEG_DT string
-,CS_WEEK_BEG_DT string
-,CAL_DATE string
-,DAY_OF_WEEK string
-,MONTH_ID string
-,PRD_DESC string
-,PRD_FLAG string
-,PRD_ID string
-,PRD_IND string
-,QTR_DESC string
-,QTR_ID string
-,QTR_IND string
-,RETAIL_WEEK string
-,RETAIL_YEAR string
-,RETAIL_START_DATE string
-,RETAIL_WK_END_DATE string
-,WEEK_IND string
-,WEEK_NUM_DESC string
-,WEEK_BEG_DATE string
-,WEEK_END_DATE string
-,WEEK_IN_YEAR_ID string
-,WEEK_ID string
-,WEEK_BEG_END_DESC_MDY string
-,WEEK_BEG_END_DESC_MD string
-,YEAR_ID string
-,YEAR_IND string
-,CAL_DT_MNS_1YEAR_DT string
-,CAL_DT_MNS_2YEAR_DT string
-,CAL_DT_MNS_1QTR_DT string
-,CAL_DT_MNS_2QTR_DT string
-,CAL_DT_MNS_1MONTH_DT string
-,CAL_DT_MNS_2MONTH_DT string
-,CAL_DT_MNS_1WEEK_DT string
-,CAL_DT_MNS_2WEEK_DT string
-,CURR_CAL_DT_MNS_1YEAR_YN_ID tinyint
-,CURR_CAL_DT_MNS_2YEAR_YN_ID tinyint
-,CURR_CAL_DT_MNS_1QTR_YN_ID tinyint
-,CURR_CAL_DT_MNS_2QTR_YN_ID tinyint
-,CURR_CAL_DT_MNS_1MONTH_YN_ID tinyint
-,CURR_CAL_DT_MNS_2MONTH_YN_ID tinyint
-,CURR_CAL_DT_MNS_1WEEK_YN_IND tinyint
-,CURR_CAL_DT_MNS_2WEEK_YN_IND tinyint
-,RTL_MONTH_OF_RTL_YEAR_ID string
-,RTL_QTR_OF_RTL_YEAR_ID tinyint
-,RTL_WEEK_OF_RTL_YEAR_ID tinyint
-,SEASON_OF_YEAR_ID tinyint
-,YTM_YN_ID tinyint
-,YTQ_YN_ID tinyint
-,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS TEXTFILE;
-
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
-
-CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
-(
-LEAF_CATEG_ID bigint
-,LEAF_CATEG_NAME string
-,SITE_ID int
-,CATEG_BUSN_MGR string
-,CATEG_BUSN_UNIT string
-,REGN_CATEG string
-,USER_DEFINED_FIELD1 string
-,USER_DEFINED_FIELD3 string
-,CRE_DATE string
-,UPD_DATE string
-,CRE_USER string
-,UPD_USER string
-,META_CATEG_ID decimal
-,META_CATEG_NAME string
-,CATEG_LVL2_ID decimal
-,CATEG_LVL3_ID decimal
-,CATEG_LVL4_ID decimal
-,CATEG_LVL5_ID decimal
-,CATEG_LVL6_ID decimal
-,CATEG_LVL7_ID decimal
-,CATEG_LVL2_NAME string
-,CATEG_LVL3_NAME string
-,CATEG_LVL4_NAME string
-,CATEG_LVL5_NAME string
-,CATEG_LVL6_NAME string
-,CATEG_LVL7_NAME string
-,CATEG_FLAGS decimal
-,ADULT_CATEG_YN string
-,DOMAIN_ID decimal
-,USER_DEFINED_FIELD5 string
-,VCS_ID decimal
-,GCS_ID decimal
-,MOVE_TO decimal
-,SAP_CATEGORY_ID decimal
-,SRC_ID tinyint
-,BSNS_VRTCL_NAME string
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS TEXTFILE;
-
-DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
-
-CREATE TABLE DEFAULT.KYLIN_SALES
-(
-TRANS_ID bigint
-,PART_DT date
-,LSTG_FORMAT_NAME string
-,LEAF_CATEG_ID bigint
-,LSTG_SITE_ID int
-,SLR_SEGMENT_CD smallint
-,PRICE decimal(19,4)
-,ITEM_COUNT bigint
-,SELLER_ID bigint
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS TEXTFILE;
-
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
\ No newline at end of file
[11/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
deleted file mode 100644
index 8f77558..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-
-public class MergeDictionaryStep extends AbstractExecutable {
-
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
-
- public MergeDictionaryStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- KylinConfig conf = context.getConfig();
- final CubeManager mgr = CubeManager.getInstance(conf);
- final CubeInstance cube = mgr.getCube(getCubeName());
- final CubeSegment newSegment = cube.getSegmentById(getSegmentId());
- final List<CubeSegment> mergingSegments = getMergingSegments(cube);
-
- Collections.sort(mergingSegments);
-
- try {
- checkLookupSnapshotsMustIncremental(mergingSegments);
-
- makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
- makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
-
- mgr.updateCube(cube);
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- } catch (IOException e) {
- logger.error("fail to merge dictionary or lookup snapshots", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- private List<CubeSegment> getMergingSegments(CubeInstance cube) {
- List<String> mergingSegmentIds = getMergingSegmentIds();
- List<CubeSegment> result = Lists.newArrayListWithCapacity(mergingSegmentIds.size());
- for (String id : mergingSegmentIds) {
- result.add(cube.getSegmentById(id));
- }
- return result;
- }
-
- private void checkLookupSnapshotsMustIncremental(List<CubeSegment> mergingSegments) {
-
- // FIXME check each newer snapshot has only NEW rows but no MODIFIED rows
- }
-
- /**
- * For the new segment, we need to create dictionaries for it, too. For
- * those dictionaries on fact table, create it by merging underlying
- * dictionaries For those dictionaries on lookup table, just copy it from
- * any one of the merging segments, it's guaranteed to be consistent(checked
- * in CubeSegmentValidator)
- *
- * @param cube
- * @param newSeg
- * @throws IOException
- */
- private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException {
- HashSet<TblColRef> colsNeedMeringDict = new HashSet<TblColRef>();
- HashSet<TblColRef> colsNeedCopyDict = new HashSet<TblColRef>();
- DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
-
- CubeDesc cubeDesc = cube.getDescriptor();
- for (DimensionDesc dim : cubeDesc.getDimensions()) {
- for (TblColRef col : dim.getColumnRefs()) {
- if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
- if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
- colsNeedMeringDict.add(col);
- } else {
- colsNeedCopyDict.add(col);
- }
- }
- }
- }
-
- for (TblColRef col : colsNeedMeringDict) {
- logger.info("Merging fact table dictionary on : " + col);
- List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
- for (CubeSegment segment : mergingSegments) {
- logger.info("Including fact table dictionary of segment : " + segment);
- if (segment.getDictResPath(col) != null) {
- DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
- dictInfos.add(dictInfo);
- }
- }
- mergeDictionaries(dictMgr, newSeg, dictInfos, col);
- }
-
- for (TblColRef col : colsNeedCopyDict) {
- String path = mergingSegments.get(0).getDictResPath(col);
- newSeg.putDictResPath(col, path);
- }
- }
-
- private DictionaryInfo mergeDictionaries(DictionaryManager dictMgr, CubeSegment cubeSeg, List<DictionaryInfo> dicts, TblColRef col) throws IOException {
- DictionaryInfo dictInfo = dictMgr.mergeDictionary(dicts);
- if (dictInfo != null)
- cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- return dictInfo;
- }
-
- /**
- * make snapshots for the new segment by copying from one of the underlying
- * merging segments. it's guaranteed to be consistent(checked in
- * CubeSegmentValidator)
- *
- * @param cube
- * @param newSeg
- */
- private void makeSnapshotForNewSegment(CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
- CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
- for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
- newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
- }
- }
-
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setMergingSegmentIds(List<String> ids) {
- setParam(MERGING_SEGMENT_IDS, StringUtils.join(ids, ","));
- }
-
- private List<String> getMergingSegmentIds() {
- final String ids = getParam(MERGING_SEGMENT_IDS);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
deleted file mode 100644
index 0c28d85..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.job.execution.Output;
-
-import java.io.IOException;
-
-/**
- * Created by qianzhou on 1/4/15.
- */
-public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
-
- private static final String SEGMENT_ID = "segmentId";
- private static final String CUBE_NAME = "cubeName";
- private static final String CONVERT_TO_HFILE_STEP_ID = "convertToHFileStepId";
- private static final String BASE_CUBOID_STEP_ID = "baseCuboidStepId";
- private static final String CREATE_FLAT_TABLE_STEP_ID = "createFlatTableStepId";
- private static final String CUBING_JOB_ID = "cubingJobId";
-
- public UpdateCubeInfoAfterBuildStep() {
- super();
- }
-
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setConvertToHFileStepId(String id) {
- setParam(CONVERT_TO_HFILE_STEP_ID, id);
- }
-
- private String getConvertToHfileStepId() {
- return getParam(CONVERT_TO_HFILE_STEP_ID);
- }
-
- public void setBaseCuboidStepId(String id) {
- setParam(BASE_CUBOID_STEP_ID, id);
- }
-
- private String getBaseCuboidStepId() {
- return getParam(BASE_CUBOID_STEP_ID);
- }
-
- public void setCreateFlatTableStepId(String id) {
- setParam(CREATE_FLAT_TABLE_STEP_ID, id);
- }
-
- public void setCubingJobId(String id) {
- setParam(CUBING_JOB_ID, id);
- }
-
- private String getCubingJobId() {
- return getParam(CUBING_JOB_ID);
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(getCubeName());
- final CubeSegment segment = cube.getSegmentById(getSegmentId());
-
- Output baseCuboidOutput = executableManager.getOutput(getBaseCuboidStepId());
- String sourceRecordsCount = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_COUNT);
- long sourceCount = 0l;
- if (StringUtils.isNotEmpty(sourceRecordsCount)) {
- sourceCount = Long.parseLong(sourceRecordsCount);
- } else {
- logger.warn("Can not get cube source record count.");
- }
-
- long sourceSize = 0l;
- String sourceRecordsSize = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_SIZE);
- if (StringUtils.isNotEmpty(sourceRecordsSize)) {
- sourceSize = Long.parseLong(sourceRecordsSize);
- } else {
- logger.warn("Can not get cube source record size.");
- }
-
- long size = 0;
- boolean segmentReady = true;
- if (!StringUtils.isBlank(getConvertToHfileStepId())) {
- String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
- if (StringUtils.isNotEmpty(cubeSizeString)) {
- size = Long.parseLong(cubeSizeString) / 1024;
- } else {
- logger.warn("Can't get cube segment size.");
- }
- } else {
- // for the increment & merge case, the increment segment is only built to be merged, won't serve query by itself
- segmentReady = false;
- }
-
- segment.setLastBuildJobID(getCubingJobId());
- segment.setLastBuildTime(System.currentTimeMillis());
- segment.setSizeKB(size);
- segment.setInputRecords(sourceCount);
- segment.setInputRecordsSize(sourceSize);
-
- try {
- if (segmentReady) {
- cubeManager.promoteNewlyBuiltSegments(cube, segment);
- } else {
- cubeManager.updateCube(cube);
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- } catch (IOException e) {
- logger.error("fail to update cube after build", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
deleted file mode 100644
index 660d38d..0000000
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by qianzhou on 1/7/15.
- */
-public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
-
- private static final String CUBE_NAME = "cubeName";
- private static final String SEGMENT_ID = "segmentId";
- private static final String MERGING_SEGMENT_IDS = "mergingSegmentIds";
- private static final String CONVERT_TO_HFILE_STEP_ID = "convertToHFileStepId";
- private static final String CUBING_JOB_ID = "cubingJobId";
-
- private final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- public UpdateCubeInfoAfterMergeStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeInstance cube = cubeManager.getCube(getCubeName());
-
- CubeSegment mergedSegment = cube.getSegmentById(getSegmentId());
- if (mergedSegment == null) {
- return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
- }
-
- long cubeSize = 0l;
- String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
- if (StringUtils.isNotEmpty(cubeSizeString)) {
- cubeSize = Long.parseLong(cubeSizeString) / 1024;
- } else {
- logger.warn("Can not get cube segment size.");
- }
-
- // collect source statistics
- List<String> mergingSegmentIds = getMergingSegmentIds();
- if (mergingSegmentIds.isEmpty()) {
- return new ExecuteResult(ExecuteResult.State.FAILED, "there are no merging segments");
- }
- long sourceCount = 0L;
- long sourceSize = 0L;
- for (String id : mergingSegmentIds) {
- CubeSegment segment = cube.getSegmentById(id);
- sourceCount += segment.getInputRecords();
- sourceSize += segment.getInputRecordsSize();
- }
-
- // update segment info
- mergedSegment.setSizeKB(cubeSize);
- mergedSegment.setInputRecords(sourceCount);
- mergedSegment.setInputRecordsSize(sourceSize);
- mergedSegment.setLastBuildJobID(getCubingJobId());
- mergedSegment.setLastBuildTime(System.currentTimeMillis());
-
- try {
- cubeManager.promoteNewlyBuiltSegments(cube, mergedSegment);
- return new ExecuteResult(ExecuteResult.State.SUCCEED);
- } catch (IOException e) {
- logger.error("fail to update cube after merge", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setSegmentId(String segmentId) {
- this.setParam(SEGMENT_ID, segmentId);
- }
-
- private String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public void setCubeName(String cubeName) {
- this.setParam(CUBE_NAME, cubeName);
- }
-
- private String getCubeName() {
- return getParam(CUBE_NAME);
- }
-
- public void setMergingSegmentIds(List<String> ids) {
- setParam(MERGING_SEGMENT_IDS, StringUtils.join(ids, ","));
- }
-
- private List<String> getMergingSegmentIds() {
- final String ids = getParam(MERGING_SEGMENT_IDS);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
- public void setConvertToHFileStepId(String id) {
- setParam(CONVERT_TO_HFILE_STEP_ID, id);
- }
-
- private String getConvertToHfileStepId() {
- return getParam(CONVERT_TO_HFILE_STEP_ID);
- }
-
- public void setCubingJobId(String id) {
- setParam(CUBING_JOB_ID, id);
- }
-
- private String getCubingJobId() {
- return getParam(CUBING_JOB_ID);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
deleted file mode 100644
index 482f7a0..0000000
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dao;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.job.exception.PersistentException;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class ExecutableDao {
-
- private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<ExecutablePO>(ExecutablePO.class);
- private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>(ExecutableOutputPO.class);
- private static final Logger logger = LoggerFactory.getLogger(ExecutableDao.class);
- private static final ConcurrentHashMap<KylinConfig, ExecutableDao> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableDao>();
-
- private ResourceStore store;
-
- public static ExecutableDao getInstance(KylinConfig config) {
- ExecutableDao r = CACHE.get(config);
- if (r == null) {
- r = new ExecutableDao(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
-
- }
- return r;
- }
-
- private ExecutableDao(KylinConfig config) {
- logger.info("Using metadata url: " + config);
- this.store = MetadataManager.getInstance(config).getStore();
- }
-
- private String pathOfJob(ExecutablePO job) {
- return pathOfJob(job.getUuid());
- }
-
- private String pathOfJob(String uuid) {
- return ResourceStore.EXECUTE_PATH_ROOT + "/" + uuid;
- }
-
- private String pathOfJobOutput(String uuid) {
- return ResourceStore.EXECUTE_OUTPUT_ROOT + "/" + uuid;
- }
-
- private ExecutablePO readJobResource(String path) throws IOException {
- return store.getResource(path, ExecutablePO.class, JOB_SERIALIZER);
- }
-
- private void writeJobResource(String path, ExecutablePO job) throws IOException {
- store.putResource(path, job, JOB_SERIALIZER);
- }
-
- private ExecutableOutputPO readJobOutputResource(String path) throws IOException {
- return store.getResource(path, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
- }
-
- private long writeJobOutputResource(String path, ExecutableOutputPO output) throws IOException {
- return store.putResource(path, output, JOB_OUTPUT_SERIALIZER);
- }
-
- public List<ExecutableOutputPO> getJobOutputs() throws PersistentException {
- try {
- ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_ROOT);
- if (resources == null || resources.isEmpty()) {
- return Collections.emptyList();
- }
- Collections.sort(resources);
- String rangeStart = resources.get(0);
- String rangeEnd = resources.get(resources.size() - 1);
- return store.getAllResources(rangeStart, rangeEnd, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
- } catch (IOException e) {
- logger.error("error get all Jobs:", e);
- throw new PersistentException(e);
- }
- }
-
- public List<ExecutablePO> getJobs() throws PersistentException {
- try {
- final List<String> jobIds = store.listResources(ResourceStore.EXECUTE_PATH_ROOT);
- if (jobIds == null || jobIds.isEmpty()) {
- return Collections.emptyList();
- }
- Collections.sort(jobIds);
- String rangeStart = jobIds.get(0);
- String rangeEnd = jobIds.get(jobIds.size() - 1);
- return store.getAllResources(rangeStart, rangeEnd, ExecutablePO.class, JOB_SERIALIZER);
- } catch (IOException e) {
- logger.error("error get all Jobs:", e);
- throw new PersistentException(e);
- }
- }
-
- public List<String> getJobIds() throws PersistentException {
- try {
- ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_PATH_ROOT);
- if (resources == null) {
- return Collections.emptyList();
- }
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(resources.size());
- for (String path : resources) {
- result.add(path.substring(path.lastIndexOf("/") + 1));
- }
- return result;
- } catch (IOException e) {
- logger.error("error get all Jobs:", e);
- throw new PersistentException(e);
- }
- }
-
- public ExecutablePO getJob(String uuid) throws PersistentException {
- try {
- return readJobResource(pathOfJob(uuid));
- } catch (IOException e) {
- logger.error("error get job:" + uuid, e);
- throw new PersistentException(e);
- }
- }
-
- public ExecutablePO addJob(ExecutablePO job) throws PersistentException {
- try {
- if (getJob(job.getUuid()) != null) {
- throw new IllegalArgumentException("job id:" + job.getUuid() + " already exists");
- }
- writeJobResource(pathOfJob(job), job);
- return job;
- } catch (IOException e) {
- logger.error("error save job:" + job.getUuid(), e);
- throw new PersistentException(e);
- }
- }
-
- public void deleteJob(String uuid) throws PersistentException {
- try {
- store.deleteResource(pathOfJob(uuid));
- } catch (IOException e) {
- logger.error("error delete job:" + uuid, e);
- throw new PersistentException(e);
- }
- }
-
- public ExecutableOutputPO getJobOutput(String uuid) throws PersistentException {
- try {
- ExecutableOutputPO result = readJobOutputResource(pathOfJobOutput(uuid));
- if (result == null) {
- result = new ExecutableOutputPO();
- result.setUuid(uuid);
- return result;
- }
- return result;
- } catch (IOException e) {
- logger.error("error get job output id:" + uuid, e);
- throw new PersistentException(e);
- }
- }
-
- public void addJobOutput(ExecutableOutputPO output) throws PersistentException {
- try {
- output.setLastModified(0);
- writeJobOutputResource(pathOfJobOutput(output.getUuid()), output);
- } catch (IOException e) {
- logger.error("error update job output id:" + output.getUuid(), e);
- throw new PersistentException(e);
- }
- }
-
- public void updateJobOutput(ExecutableOutputPO output) throws PersistentException {
- try {
- final long ts = writeJobOutputResource(pathOfJobOutput(output.getUuid()), output);
- output.setLastModified(ts);
- } catch (IOException e) {
- logger.error("error update job output id:" + output.getUuid(), e);
- throw new PersistentException(e);
- }
- }
-
- public void deleteJobOutput(String uuid) throws PersistentException {
- try {
- store.deleteResource(pathOfJobOutput(uuid));
- } catch (IOException e) {
- logger.error("error delete job:" + uuid, e);
- throw new PersistentException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
deleted file mode 100644
index f086558..0000000
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutableOutputPO.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dao;
-
-import java.util.Map;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Maps;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ExecutableOutputPO extends RootPersistentEntity {
-
- @JsonProperty("content")
- private String content;
-
- @JsonProperty("status")
- private String status = "READY";
-
- @JsonProperty("info")
- private Map<String, String> info = Maps.newHashMap();
-
- public String getContent() {
- return content;
- }
-
- public void setContent(String content) {
- this.content = content;
- }
-
- public String getStatus() {
- return status;
- }
-
- public void setStatus(String status) {
- this.status = status;
- }
-
- public Map<String, String> getInfo() {
- return info;
- }
-
- public void setInfo(Map<String, String> info) {
- this.info = info;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java b/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
deleted file mode 100644
index 97b8d0b..0000000
--- a/job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dao;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Maps;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ExecutablePO extends RootPersistentEntity {
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("tasks")
- private List<ExecutablePO> tasks;
-
- @JsonProperty("type")
- private String type;
-
- @JsonProperty("params")
- private Map<String, String> params = Maps.newHashMap();
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public List<ExecutablePO> getTasks() {
- return tasks;
- }
-
- public void setTasks(List<ExecutablePO> tasks) {
- this.tasks = tasks;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public Map<String, String> getParams() {
- return params;
- }
-
- public void setParams(Map<String, String> params) {
- this.params = params;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
deleted file mode 100644
index 8edc8a0..0000000
--- a/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.engine;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.tools.OptionsHelper;
-import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author ysong1
- */
-public class JobEngineConfig {
- private static final Logger logger = LoggerFactory.getLogger(JobEngineConfig.class);
- public static String HADOOP_JOB_CONF_FILENAME = "kylin_job_conf";
-
- private static File getJobConfig(String fileName) {
- String path = System.getProperty(KylinConfig.KYLIN_CONF);
- if (StringUtils.isNotEmpty(path)) {
- return new File(path, fileName);
- }
-
- path = KylinConfig.getKylinHome();
- if (StringUtils.isNotEmpty(path)) {
- return new File(path + File.separator + "conf", fileName);
- }
- return null;
- }
-
- private String getHadoopJobConfFilePath(RealizationCapacity capaticy, boolean appendSuffix) throws IOException {
- String hadoopJobConfFile;
- if (capaticy != null && appendSuffix) {
- hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + capaticy.toString().toLowerCase() + ".xml");
- } else {
- hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml");
- }
-
- File jobConfig = getJobConfig(hadoopJobConfFile);
- if (jobConfig == null || !jobConfig.exists()) {
- logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
- jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml");
- if (jobConfig == null || !jobConfig.exists()) {
- logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
- throw new RuntimeException("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
- }
- }
- return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath());
- }
-
- public String getHadoopJobConfFilePath(RealizationCapacity capaticy) throws IOException {
- String path = getHadoopJobConfFilePath(capaticy, true);
- if (!StringUtils.isEmpty(path)) {
- logger.info("Chosen job conf is : " + path);
- return path;
- } else {
- path = getHadoopJobConfFilePath(capaticy, false);
- if (!StringUtils.isEmpty(path)) {
- logger.info("Chosen job conf is : " + path);
- return path;
- }
- }
- return "";
- }
-
- private void inputStreamToFile(InputStream ins, File file) throws IOException {
- OutputStream os = new FileOutputStream(file);
- int bytesRead = 0;
- byte[] buffer = new byte[8192];
- while ((bytesRead = ins.read(buffer, 0, 8192)) != -1) {
- os.write(buffer, 0, bytesRead);
- }
- os.close();
- ins.close();
- }
-
- // there should be no setters
- private final KylinConfig config;
-
- public JobEngineConfig(KylinConfig config) {
- this.config = config;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public String getHdfsWorkingDirectory() {
- return config.getHdfsWorkingDirectory();
- }
-
- /**
- * @return the maxConcurrentJobLimit
- */
- public int getMaxConcurrentJobLimit() {
- return config.getMaxConcurrentJobLimit();
- }
-
- /**
- * @return the timeZone
- */
- public String getTimeZone() {
- return config.getTimeZone();
- }
-
- /**
- * @return the adminDls
- */
- public String getAdminDls() {
- return config.getAdminDls();
- }
-
- /**
- * @return the jobStepTimeout
- */
- public long getJobStepTimeout() {
- return config.getJobStepTimeout();
- }
-
- /**
- * @return the asyncJobCheckInterval
- */
- public int getAsyncJobCheckInterval() {
- return config.getYarnStatusCheckIntervalSeconds();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((config == null) ? 0 : config.hashCode());
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- JobEngineConfig other = (JobEngineConfig) obj;
- if (config == null) {
- if (other.config != null)
- return false;
- } else if (!config.equals(other.config))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java b/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
deleted file mode 100644
index 7956fc0..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/ExecuteException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public class ExecuteException extends Exception {
-
- private static final long serialVersionUID = 5677121412192984281L;
-
- public ExecuteException() {
- }
-
- public ExecuteException(String message) {
- super(message);
- }
-
- public ExecuteException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ExecuteException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java b/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
deleted file mode 100644
index d57d1b3..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * Created by qianzhou on 12/26/14.
- */
-public class IllegalStateTranferException extends RuntimeException {
-
- private static final long serialVersionUID = 8466551519300132702L;
-
- public IllegalStateTranferException() {
- }
-
- public IllegalStateTranferException(String message) {
- super(message);
- }
-
- public IllegalStateTranferException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public IllegalStateTranferException(Throwable cause) {
- super(cause);
- }
-
- public IllegalStateTranferException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/JobException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/JobException.java b/job/src/main/java/org/apache/kylin/job/exception/JobException.java
deleted file mode 100644
index 9b6cef6..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/JobException.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * @author xduo
- *
- */
-public class JobException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- /**
- *
- */
- public JobException() {
- super();
- }
-
- /**
- * @param message
- * @param cause
- */
- public JobException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * @param message
- */
- public JobException(String message) {
- super(message);
- }
-
- /**
- * @param cause
- */
- public JobException(Throwable cause) {
- super(cause);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/LockException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/LockException.java b/job/src/main/java/org/apache/kylin/job/exception/LockException.java
deleted file mode 100644
index 67568fc..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/LockException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * Created by qianzhou on 12/17/14.
- */
-public class LockException extends Exception {
- private static final long serialVersionUID = 2072745879281754945L;
-
- public LockException() {
- }
-
- public LockException(String message) {
- super(message);
- }
-
- public LockException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public LockException(Throwable cause) {
- super(cause);
- }
-
- public LockException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java b/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
deleted file mode 100644
index 60401b5..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/PersistentException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public class PersistentException extends Exception {
- private static final long serialVersionUID = -4239863858506718998L;
-
- public PersistentException() {
- }
-
- public PersistentException(String message) {
- super(message);
- }
-
- public PersistentException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public PersistentException(Throwable cause) {
- super(cause);
- }
-
- public PersistentException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java b/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
deleted file mode 100644
index 190db6e..0000000
--- a/job/src/main/java/org/apache/kylin/job/exception/SchedulerException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.exception;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public class SchedulerException extends Exception {
- private static final long serialVersionUID = 349041244824274861L;
-
- public SchedulerException() {
- }
-
- public SchedulerException(String message) {
- super(message);
- }
-
- public SchedulerException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SchedulerException(Throwable cause) {
- super(cause);
- }
-
- public SchedulerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
deleted file mode 100644
index f7e4332..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LogTitlePrinter;
-import org.apache.kylin.common.util.MailService;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.impl.threadpool.DefaultContext;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- */
-public abstract class AbstractExecutable implements Executable, Idempotent {
-
- protected static final String SUBMITTER = "submitter";
- protected static final String NOTIFY_LIST = "notify_list";
- protected static final String START_TIME = "startTime";
- protected static final String END_TIME = "endTime";
-
- protected static final Logger logger = LoggerFactory.getLogger(AbstractExecutable.class);
-
- private String name;
- private String id;
- private Map<String, String> params = Maps.newHashMap();
-
- protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- public AbstractExecutable() {
- setId(UUID.randomUUID().toString());
- }
-
- protected void onExecuteStart(ExecutableContext executableContext) {
- Map<String, String> info = Maps.newHashMap();
- info.put(START_TIME, Long.toString(System.currentTimeMillis()));
- executableManager.updateJobOutput(getId(), ExecutableState.RUNNING, info, null);
- }
-
- protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
- setEndTime(System.currentTimeMillis());
- if (!isDiscarded()) {
- if (result.succeed()) {
- executableManager.updateJobOutput(getId(), ExecutableState.SUCCEED, null, result.output());
- } else {
- executableManager.updateJobOutput(getId(), ExecutableState.ERROR, null, result.output());
- }
- } else {
- }
- }
-
- protected void onExecuteError(Throwable exception, ExecutableContext executableContext) {
- if (!isDiscarded()) {
- executableManager.addJobInfo(getId(), END_TIME, Long.toString(System.currentTimeMillis()));
- String output = null;
- if (exception != null) {
- final StringWriter out = new StringWriter();
- exception.printStackTrace(new PrintWriter(out));
- output = out.toString();
- }
- executableManager.updateJobOutput(getId(), ExecutableState.ERROR, null, output);
- } else {
- }
- }
-
- @Override
- public final ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException {
-
- //print a eye-catching title in log
- LogTitlePrinter.printTitle(this.getName());
-
- Preconditions.checkArgument(executableContext instanceof DefaultContext);
- ExecuteResult result;
- try {
- onExecuteStart(executableContext);
- result = doWork(executableContext);
- } catch (Throwable e) {
- logger.error("error running Executable", e);
- onExecuteError(e, executableContext);
- throw new ExecuteException(e);
- }
- onExecuteFinished(result, executableContext);
- return result;
- }
-
- protected abstract ExecuteResult doWork(ExecutableContext context) throws ExecuteException;
-
- @Override
- public void cleanup() throws ExecuteException {
-
- }
-
- @Override
- public boolean isRunnable() {
- return this.getStatus() == ExecutableState.READY;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @Override
- public final String getId() {
- return this.id;
- }
-
- public final void setId(String id) {
- this.id = id;
- }
-
- @Override
- public final ExecutableState getStatus() {
- return executableManager.getOutput(this.getId()).getState();
- }
-
- @Override
- public final Map<String, String> getParams() {
- return Collections.unmodifiableMap(this.params);
- }
-
- public final String getParam(String key) {
- return this.params.get(key);
- }
-
- public final void setParam(String key, String value) {
- this.params.put(key, value);
- }
-
- public final void setParams(Map<String, String> params) {
- this.params.putAll(params);
- }
-
- public final long getLastModified() {
- return executableManager.getOutput(getId()).getLastModified();
- }
-
- public final void setSubmitter(String submitter) {
- setParam(SUBMITTER, submitter);
- }
-
- public final List<String> getNotifyList() {
- final String str = getParam(NOTIFY_LIST);
- if (str != null) {
- return Lists.newArrayList(StringUtils.split(str, ","));
- } else {
- return Collections.emptyList();
- }
- }
-
- public final void setNotifyList(String notifications) {
- setParam(NOTIFY_LIST, notifications);
- }
-
- public final void setNotifyList(List<String> notifications) {
- setNotifyList(StringUtils.join(notifications, ","));
- }
-
- protected Pair<String, String> formatNotifications(ExecutableState state) {
- return null;
- }
-
- protected final void notifyUserStatusChange(ExecutableState state) {
- try {
- List<String> users = Lists.newArrayList();
- users.addAll(getNotifyList());
- final String adminDls = KylinConfig.getInstanceFromEnv().getAdminDls();
- if (null != adminDls) {
- for (String adminDl : adminDls.split(",")) {
- users.add(adminDl);
- }
- }
- if (users.isEmpty()) {
- return;
- }
- final Pair<String, String> email = formatNotifications(state);
- if (email == null) {
- return;
- }
- logger.info("prepare to send email to:" + users);
- logger.info("job name:" + getName());
- logger.info("submitter:" + getSubmitter());
- logger.info("notify list:" + users);
- new MailService().sendMail(users, email.getLeft(), email.getRight());
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
-
- public final String getSubmitter() {
- return getParam(SUBMITTER);
- }
-
- @Override
- public final Output getOutput() {
- return executableManager.getOutput(getId());
- }
-
- protected long getExtraInfoAsLong(String key, long defaultValue) {
- return getExtraInfoAsLong(executableManager.getOutput(getId()), key, defaultValue);
- }
-
- public static long getStartTime(Output output) {
- return getExtraInfoAsLong(output, START_TIME, 0L);
- }
-
- public static long getEndTime(Output output) {
- return getExtraInfoAsLong(output, END_TIME, 0L);
- }
-
- public static long getDuration(long startTime, long endTime) {
- if (startTime == 0) {
- return 0;
- }
- if (endTime == 0) {
- return System.currentTimeMillis() - startTime;
- } else {
- return endTime - startTime;
- }
- }
-
- public static String getExtraInfo(Output output, String key) {
- return output.getExtra().get(key);
- }
-
- public static long getExtraInfoAsLong(Output output, String key, long defaultValue) {
- final String str = output.getExtra().get(key);
- if (str != null) {
- return Long.parseLong(str);
- } else {
- return defaultValue;
- }
- }
-
- protected final void addExtraInfo(String key, String value) {
- executableManager.addJobInfo(getId(), key, value);
- }
-
- public final void setStartTime(long time) {
- addExtraInfo(START_TIME, time + "");
- }
-
- public final void setEndTime(long time) {
- addExtraInfo(END_TIME, time + "");
- }
-
- public final long getStartTime() {
- return getExtraInfoAsLong(START_TIME, 0L);
- }
-
- public final long getEndTime() {
- return getExtraInfoAsLong(END_TIME, 0L);
- }
-
- public final long getDuration() {
- return getDuration(getStartTime(), getEndTime());
- }
-
- /*
- * discarded is triggered by JobService, the Scheduler is not awake of that
- *
- * */
- protected final boolean isDiscarded() {
- final ExecutableState status = executableManager.getOutput(getId()).getState();
- return status == ExecutableState.DISCARDED;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()).toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
deleted file mode 100644
index f372f38..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/ChainedExecutable.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.List;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public interface ChainedExecutable extends Executable {
-
- List<? extends AbstractExecutable> getTasks();
-
- void addTask(AbstractExecutable executable);
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
deleted file mode 100644
index 61140b5..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.manager.ExecutableManager;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Created by qianzhou on 12/16/14.
- */
-public class DefaultChainedExecutable extends AbstractExecutable implements ChainedExecutable {
-
- private final List<AbstractExecutable> subTasks = Lists.newArrayList();
-
- protected final ExecutableManager jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- public DefaultChainedExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- List<? extends Executable> executables = getTasks();
- final int size = executables.size();
- for (int i = 0; i < size; ++i) {
- Executable subTask = executables.get(i);
- if (subTask.isRunnable()) {
- return subTask.execute(context);
- }
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED, null);
- }
-
- @Override
- protected void onExecuteStart(ExecutableContext executableContext) {
- Map<String, String> info = Maps.newHashMap();
- info.put(START_TIME, Long.toString(System.currentTimeMillis()));
- final long startTime = getStartTime();
- if (startTime > 0) {
- jobService.updateJobOutput(getId(), ExecutableState.RUNNING, null, null);
- } else {
- jobService.updateJobOutput(getId(), ExecutableState.RUNNING, info, null);
- }
- }
-
- @Override
- protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
- if (isDiscarded()) {
- setEndTime(System.currentTimeMillis());
- notifyUserStatusChange(ExecutableState.DISCARDED);
- } else if (result.succeed()) {
- List<? extends Executable> jobs = getTasks();
- boolean allSucceed = true;
- boolean hasError = false;
- for (Executable task : jobs) {
- final ExecutableState status = task.getStatus();
- if (status == ExecutableState.ERROR) {
- hasError = true;
- }
- if (status != ExecutableState.SUCCEED) {
- allSucceed = false;
- }
- }
- if (allSucceed) {
- setEndTime(System.currentTimeMillis());
- jobService.updateJobOutput(getId(), ExecutableState.SUCCEED, null, null);
- notifyUserStatusChange(ExecutableState.SUCCEED);
- } else if (hasError) {
- setEndTime(System.currentTimeMillis());
- jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, null);
- notifyUserStatusChange(ExecutableState.ERROR);
- } else {
- jobService.updateJobOutput(getId(), ExecutableState.READY, null, null);
- }
- } else {
- setEndTime(System.currentTimeMillis());
- jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, null);
- notifyUserStatusChange(ExecutableState.ERROR);
- }
- }
-
- @Override
- public List<AbstractExecutable> getTasks() {
- return subTasks;
- }
-
- public final AbstractExecutable getTaskByName(String name) {
- for (AbstractExecutable task : subTasks) {
- if (task.getName() != null && task.getName().equalsIgnoreCase(name)) {
- return task;
- }
- }
- return null;
- }
-
- public void addTask(AbstractExecutable executable) {
- executable.setId(getId() + "-" + String.format("%02d", subTasks.size()));
- this.subTasks.add(executable);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java b/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
deleted file mode 100644
index a2ee08e..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/DefaultOutput.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-
-/**
- * Created by qianzhou on 1/6/15.
- */
-public class DefaultOutput implements Output {
-
- private ExecutableState state;
- private Map<String, String> extra;
- private String verboseMsg;
- private long lastModified;
-
- @Override
- public Map<String, String> getExtra() {
- return extra;
- }
-
- @Override
- public String getVerboseMsg() {
- return verboseMsg;
- }
-
- @Override
- public ExecutableState getState() {
- return state;
- }
-
- @Override
- public long getLastModified() {
- return lastModified;
- }
-
- public void setState(ExecutableState state) {
- this.state = state;
- }
-
- public void setExtra(Map<String, String> extra) {
- this.extra = extra;
- }
-
- public void setVerboseMsg(String verboseMsg) {
- this.verboseMsg = verboseMsg;
- }
-
- public void setLastModified(long lastModified) {
- this.lastModified = lastModified;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int hashCode = state.hashCode();
- hashCode = hashCode * prime + extra.hashCode();
- hashCode = hashCode * prime + verboseMsg.hashCode();
- hashCode = hashCode * prime + Long.valueOf(lastModified).hashCode();
- return hashCode;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof DefaultOutput)) {
- return false;
- }
- DefaultOutput another = ((DefaultOutput) obj);
- if (this.state != another.state) {
- return false;
- }
- if (!extra.equals(another.extra)) {
- return false;
- }
- if (this.lastModified != another.lastModified) {
- return false;
- }
- return StringUtils.equals(verboseMsg, another.verboseMsg);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/Executable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Executable.java b/job/src/main/java/org/apache/kylin/job/execution/Executable.java
deleted file mode 100644
index 2131655..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/Executable.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.Map;
-
-import org.apache.kylin.job.exception.ExecuteException;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public interface Executable {
-
- String getId();
-
- String getName();
-
- ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException;
-
- ExecutableState getStatus();
-
- Output getOutput();
-
- boolean isRunnable();
-
- Map<String, String> getParams();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java b/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
deleted file mode 100644
index 2886893..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import org.apache.kylin.common.KylinConfig;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public interface ExecutableContext {
-
- Object getSchedulerContext();
-
- KylinConfig getConfig();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java b/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
deleted file mode 100644
index 95089d2..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
-
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public enum ExecutableState {
-
- READY, RUNNING, ERROR, STOPPED, DISCARDED, SUCCEED;
-
- private static Multimap<ExecutableState, ExecutableState> VALID_STATE_TRANSFER;
-
- static {
- VALID_STATE_TRANSFER = Multimaps.newSetMultimap(Maps.<ExecutableState, Collection<ExecutableState>> newEnumMap(ExecutableState.class), new Supplier<Set<ExecutableState>>() {
- @Override
- public Set<ExecutableState> get() {
- return new CopyOnWriteArraySet<ExecutableState>();
- }
- });
-
- //scheduler
- VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.RUNNING);
- VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.ERROR);
- //user
- VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.DISCARDED);
-
- //job
- VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.READY);
- //job
- VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.SUCCEED);
- //user
- VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.DISCARDED);
- //scheduler,job
- VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.ERROR);
-
- VALID_STATE_TRANSFER.put(ExecutableState.STOPPED, ExecutableState.DISCARDED);
- VALID_STATE_TRANSFER.put(ExecutableState.STOPPED, ExecutableState.READY);
-
- VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.DISCARDED);
- VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.READY);
- }
-
- public boolean isFinalState() {
- return this == SUCCEED || this == DISCARDED;
- }
-
- public static boolean isValidStateTransfer(ExecutableState from, ExecutableState to) {
- return VALID_STATE_TRANSFER.containsEntry(from, to);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java b/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
deleted file mode 100644
index 27a2407..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public final class ExecuteResult {
-
- public static enum State {
- SUCCEED, FAILED, ERROR, DISCARDED, STOPPED
- }
-
- private final State state;
- private final String output;
-
- public ExecuteResult(State state) {
- this(state, "");
- }
-
- public ExecuteResult(State state, String output) {
- Preconditions.checkArgument(state != null, "state cannot be null");
- this.state = state;
- this.output = output;
- }
-
- public State state() {
- return state;
- }
-
- public boolean succeed() {
- return state == State.SUCCEED;
- }
-
- public String output() {
- return output;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java b/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
deleted file mode 100644
index 6515343..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/Idempotent.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import org.apache.kylin.job.exception.ExecuteException;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public interface Idempotent {
-
- void cleanup() throws ExecuteException;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/execution/Output.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/execution/Output.java b/job/src/main/java/org/apache/kylin/job/execution/Output.java
deleted file mode 100644
index f77c71a..0000000
--- a/job/src/main/java/org/apache/kylin/job/execution/Output.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-import java.util.Map;
-
-/**
- * Created by qianzhou on 1/6/15.
- */
-public interface Output {
-
- Map<String, String> getExtra();
-
- String getVerboseMsg();
-
- ExecutableState getState();
-
- long getLastModified();
-}
[02/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/test_cal_dt/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/test_cal_dt/part-r-00000 b/job/src/test/resources/data/test_cal_dt/part-r-00000
deleted file mode 100644
index 206c349..0000000
--- a/job/src/test/resources/data/test_cal_dt/part-r-00000
+++ /dev/null
@@ -1,366 +0,0 @@
-2013-08-16,2013-01-01,2013-07-01,2013-08-01,2013-08-11,0,-1,-3,-15,-103,0,-1,-4,-15,-15,41501,228,47,16,6,33,5928,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-17,16-Aug-2013,Aug 16th 2013,Fri 08-16-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-11,2013-08-12,2013-08-16,Fri ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,33,2013,2013-08-11,2013-08-17,N,Wk.33 - 13,2013-08-11 00:00:00,2013-08-17 00:00:00,2013W33 ,2013W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2013,N,2012-08-16,2011-08-16,2013-05-16,2013-02-16,2013-07-16,2013-06-16,2013-08-09,2013-08-02,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-03,2013-01-01,2013-01-01,2013-01-01,2013-01-01,0,-3,-10,-47,-328,0,-3,-11,-47,-47,41276,3,3,3,5,1,5896,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-05,03-Jan-2013,Jan 3rd 2013,Thu 01-03-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2013-01-03,Thu ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,1,2013,2012-12-30,2013-01-05,N,Wk.01 - 13,2013-01-01 00:00:00,2013-01-05 00:00:00,2013W01 ,2013W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2013,N,2012-01-03,2011-01-03,2012-10-03,2012-07-03,2012-12-03,2012-11-03,2012-12-27,2012-12-20,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-10,2013-01-01,2013-04-01,2013-04-01,2013-04-07,0,-2,-7,-33,-231,0,-2,-8,-33,-33,41373,100,10,10,4,15,5910,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-13,10-Apr-2013,Apr 10th 2013,Wed 04-10-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-07,2013-04-08,2013-04-10,Wed ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,15,2013,2013-04-07,2013-04-13,N,Wk.15 - 13,2013-04-07 00:00:00,2013-04-13 00:00:00,2013W15 ,2013W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2013,N,2012-04-10,2011-04-10,2013-01-10,2012-10-10,2013-03-10,2013-02-10,2013-04-03,2013-03-27,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-06-12,2013-01-01,2013-04-01,2013-06-01,2013-06-09,0,-2,-5,-24,-168,0,-2,-6,-24,-24,41436,163,73,12,4,24,5919,3,6,1362,2,454,114,2013-12-31,2013-06-30,2013-06-30,2013-06-15,12-Jun-2013,Jun 12th 2013,Wed 06-12-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-05-26,2013-06-09,2013-06-10,2013-06-12,Wed ,2013M06,Jun-2013,N,2013M06 ,N,Year 2013 - Quarter 02,2013Q02 ,N,24,2013,2013-06-09,2013-06-15,N,Wk.24 - 13,2013-06-09 00:00:00,2013-06-15 00:00:00,2013W24 ,2013W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2013,N,2012-06-12,2011-06-12,2013-03-12,2012-12-12,2013-05-12,2013-04-12,2013-06-05,2013-05-29,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-27,2013-01-01,2013-04-01,2013-04-01,2013-04-21,0,-2,-7,-31,-214,0,-2,-8,-31,-31,41390,117,27,27,7,17,5912,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-27,27-Apr-2013,Apr 27th 2013,Sat 04-27-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-21,2013-04-22,2013-04-27,Sat ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,17,2013,2013-04-21,2013-04-27,N,Wk.17 - 13,2013-04-21 00:00:00,2013-04-27 00:00:00,2013W17 ,2013W17 ,04/21/13 - 04/27/13,04/21 - 04/27,2013,N,2012-04-27,2011-04-27,2013-01-27,2012-10-27,2013-03-27,2013-02-27,2013-04-20,2013-04-13,0,0,0,0,0,0,0,0,4,2,17,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-19,2013-01-01,2013-07-01,2013-09-01,2013-09-15,0,-1,-2,-10,-69,0,-1,-3,-10,-10,41535,262,81,19,5,38,5933,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-21,19-Sep-2013,Sep 19th 2013,Thu 09-19-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-15,2013-09-16,2013-09-19,Thu ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,38,2013,2013-09-15,2013-09-21,N,Wk.38 - 13,2013-09-15 00:00:00,2013-09-21 00:00:00,2013W38 ,2013W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2013,N,2012-09-19,2011-09-19,2013-06-19,2013-03-19,2013-08-19,2013-07-19,2013-09-12,2013-09-05,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-07,2013-01-01,2013-01-01,2013-03-01,2013-03-03,0,-3,-8,-38,-265,0,-3,-9,-38,-38,41339,66,66,7,5,10,5905,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-09,07-Mar-2013,Mar 7th 2013,Thu 03-07-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-03,2013-03-04,2013-03-07,Thu ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,10,2013,2013-03-03,2013-03-09,N,Wk.10 - 13,2013-03-03 00:00:00,2013-03-09 00:00:00,2013W10 ,2013W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2013,N,2012-03-07,2011-03-07,2012-12-07,2012-09-07,2013-02-07,2013-01-07,2013-02-28,2013-02-21,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-28,2013-01-01,2013-01-01,2013-03-01,2013-03-24,0,-3,-8,-35,-244,0,-3,-9,-35,-35,41360,87,87,28,5,13,5908,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-30,28-Mar-2013,Mar 28th 2013,Thu 03-28-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-24,2013-03-25,2013-03-28,Thu ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,13,2013,2013-03-24,2013-03-30,N,Wk.13 - 13,2013-03-24 00:00:00,2013-03-30 00:00:00,2013W13 ,2013W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2013,N,2012-03-28,2011-03-28,2012-12-28,2012-09-28,2013-02-28,2013-01-28,2013-03-21,2013-03-14,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-23,2013-01-01,2013-10-01,2013-11-01,2013-11-17,0,0,0,-1,-4,0,0,-1,-1,-1,41600,327,54,23,7,47,5942,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-23,23-Nov-2013,Nov 23rd 2013,Sat 11-23-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-17,2013-11-18,2013-11-23,Sat ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,47,2013,2013-11-17,2013-11-23,N,Wk.47 - 13,2013-11-17 00:00:00,2013-11-23 00:00:00,2013W47 ,2013W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2013,N,2012-11-23,2011-11-23,2013-08-23,2013-05-23,2013-10-23,2013-09-23,2013-11-16,2013-11-09,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-09,2013-01-01,2013-01-01,2013-03-01,2013-03-03,0,-3,-8,-38,-263,0,-3,-9,-38,-38,41341,68,68,9,7,10,5905,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-09,09-Mar-2013,Mar 9th 2013,Sat 03-09-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-03,2013-03-04,2013-03-09,Sat ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,10,2013,2013-03-03,2013-03-09,N,Wk.10 - 13,2013-03-03 00:00:00,2013-03-09 00:00:00,2013W10 ,2013W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2013,N,2012-03-09,2011-03-09,2012-12-09,2012-09-09,2013-02-09,2013-01-09,2013-03-02,2013-02-23,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-21,2013-01-01,2013-10-01,2013-11-01,2013-11-17,0,0,0,-1,-6,0,0,-1,-1,-1,41598,325,52,21,5,47,5942,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-23,21-Nov-2013,Nov 21st 2013,Thu 11-21-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-17,2013-11-18,2013-11-21,Thu ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,47,2013,2013-11-17,2013-11-23,N,Wk.47 - 13,2013-11-17 00:00:00,2013-11-23 00:00:00,2013W47 ,2013W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2013,N,2012-11-21,2011-11-21,2013-08-21,2013-05-21,2013-10-21,2013-09-21,2013-11-14,2013-11-07,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-26,2013-01-01,2013-01-01,2013-03-01,2013-03-24,0,-3,-8,-35,-246,0,-3,-9,-35,-35,41358,85,85,26,3,13,5908,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-30,26-Mar-2013,Mar 26th 2013,Tue 03-26-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-24,2013-03-25,2013-03-26,Tue ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,13,2013,2013-03-24,2013-03-30,N,Wk.13 - 13,2013-03-24 00:00:00,2013-03-30 00:00:00,2013W13 ,2013W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2013,N,2012-03-26,2011-03-26,2012-12-26,2012-09-26,2013-02-26,2013-01-26,2013-03-19,2013-03-12,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-24,2013-01-01,2013-01-01,2013-01-01,2013-01-20,0,-3,-10,-44,-307,0,-3,-11,-44,-44,41297,24,24,24,5,4,5899,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-26,24-Jan-2013,Jan 24th 2013,Thu 01-24-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2013-01-20,2013-01-21,2013-01-24,Thu ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,4,2013,2013-01-20,2013-01-26,N,Wk.04 - 13,2013-01-20 00:00:00,2013-01-26 00:00:00,2013W04 ,2013W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2013,N,2012-01-24,2011-01-24,2012-10-24,2012-07-24,2012-12-24,2012-11-24,2013-01-17,2013-01-10,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-01,2013-01-01,2013-01-01,2013-01-01,2013-01-01,0,-3,-10,-47,-330,0,-3,-11,-47,-47,41274,1,1,1,3,1,5896,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-05,01-Jan-2013,Jan 1st 2013,Tue 01-01-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2013-01-01,Tue ,2013M01,Jan-2013,Y,2013M01 ,Y,Year 2013 - Quarter 01,2013Q01 ,Y,1,2013,2012-12-30,2013-01-05,Y,Wk.01 - 13,2013-01-01 00:00:00,2013-01-05 00:00:00,2013W01 ,2013W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2013,Y,2012-01-01,2011-01-01,2012-10-01,2012-07-01,2012-12-01,2012-11-01,2012-12-25,2012-12-18,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-29,2013-01-01,2013-04-01,2013-04-01,2013-04-28,0,-2,-7,-30,-212,0,-2,-7,-30,-30,41392,119,29,29,2,18,5913,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-05-04,29-Apr-2013,Apr 29th 2013,Mon 04-29-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-04-28,2013-04-28,2013-04-29,2013-04-29,Mon ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,18,2013,2013-04-28,2013-05-04,N,Wk.18 - 13,2013-04-28 00:00:00,2013-05-04 00:00:00,2013W18 ,2013W18 ,04/28/13 - 05/04/13,04/28 - 05/04,2013,N,2012-04-29,2011-04-29,2013-01-29,2012-10-29,2013-03-29,2013-02-28,2013-04-22,2013-04-15,0,0,0,0,0,0,0,0,5,2,18,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-11,2013-01-01,2013-04-01,2013-05-01,2013-05-05,0,-2,-6,-29,-200,0,-2,-7,-29,-29,41404,131,41,11,7,19,5914,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-11,11-May-2013,May 11th 2013,Sat 05-11-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-05,2013-05-06,2013-05-11,Sat ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,19,2013,2013-05-05,2013-05-11,N,Wk.19 - 13,2013-05-05 00:00:00,2013-05-11 00:00:00,2013W19 ,2013W19 ,05/05/13 - 05/11/13,05/05 - 05/11,2013,N,2012-05-11,2011-05-11,2013-02-11,2012-11-11,2013-04-11,2013-03-11,2013-05-04,2013-04-27,0,0,0,0,0,0,0,0,5,2,19,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-01,2013-01-01,2013-10-01,2013-10-01,2013-09-29,0,0,-1,-8,-57,0,0,-2,-8,-8,41547,274,1,1,3,40,5935,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-05,01-Oct-2013,Oct 1st 2013,Tue 10-01-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-09-29,2013-09-30,2013-10-01,Tue ,2013M10,Oct-2013,Y,2013M10 ,Y,Year 2013 - Quarter 04,2013Q04 ,Y,40,2013,2013-09-29,2013-10-05,N,Wk.40 - 13,2013-09-29 00:00:00,2013-10-05 00:00:00,2013W40 ,2013W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2013,N,2012-10-01,2011-10-01,2013-07-01,2013-04-01,2013-09-01,2013-08-01,2013-09-24,2013-09-17,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-05,2013-01-01,2013-10-01,2013-12-01,2013-12-01,0,0,1,1,8,0,0,0,1,1,41612,339,66,5,5,49,5944,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-07,05-Dec-2013,Dec 5th 2013,Thu 12-05-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-01,2013-12-02,2013-12-05,Thu ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,49,2013,2013-12-01,2013-12-07,N,Wk.49 - 13,2013-12-01 00:00:00,2013-12-07 00:00:00,2013W49 ,2013W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2013,N,2012-12-05,2011-12-05,2013-09-05,2013-06-05,2013-11-05,2013-10-05,2013-11-28,2013-11-21,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-05,2013-01-01,2013-01-01,2013-03-01,2013-03-03,0,-3,-8,-38,-267,0,-3,-9,-38,-38,41337,64,64,5,3,10,5905,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-09,05-Mar-2013,Mar 5th 2013,Tue 03-05-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-03,2013-03-04,2013-03-05,Tue ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,10,2013,2013-03-03,2013-03-09,N,Wk.10 - 13,2013-03-03 00:00:00,2013-03-09 00:00:00,2013W10 ,2013W10 ,03/03/13 - 03/09/13,03/03 - 03/09,2013,N,2012-03-05,2011-03-05,2012-12-05,2012-09-05,2013-02-05,2013-01-05,2013-02-26,2013-02-19,0,0,0,0,0,0,0,0,3,1,10,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-03,2013-01-01,2013-10-01,2013-10-01,2013-09-29,0,0,-1,-8,-55,0,0,-2,-8,-8,41549,276,3,3,5,40,5935,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-05,03-Oct-2013,Oct 3rd 2013,Thu 10-03-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-09-29,2013-09-30,2013-10-03,Thu ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,40,2013,2013-09-29,2013-10-05,N,Wk.40 - 13,2013-09-29 00:00:00,2013-10-05 00:00:00,2013W40 ,2013W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2013,N,2012-10-03,2011-10-03,2013-07-03,2013-04-03,2013-09-03,2013-08-03,2013-09-26,2013-09-19,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-22,2013-01-01,2013-01-01,2013-01-01,2013-01-20,0,-3,-10,-44,-309,0,-3,-11,-44,-44,41295,22,22,22,3,4,5899,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-26,22-Jan-2013,Jan 22nd 2013,Tue 01-22-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2013-01-20,2013-01-21,2013-01-22,Tue ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,4,2013,2013-01-20,2013-01-26,N,Wk.04 - 13,2013-01-20 00:00:00,2013-01-26 00:00:00,2013W04 ,2013W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2013,N,2012-01-22,2011-01-22,2012-10-22,2012-07-22,2012-12-22,2012-11-22,2013-01-15,2013-01-08,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-26,2013-01-01,2013-10-01,2013-12-01,2013-12-22,0,0,1,4,29,0,0,0,4,4,41633,360,87,26,5,52,5947,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-28,26-Dec-2013,Dec 26th 2013,Thu 12-26-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-22,2013-12-23,2013-12-26,Thu ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,52,2013,2013-12-22,2013-12-28,N,Wk.52 - 13,2013-12-22 00:00:00,2013-12-28 00:00:00,2013W52 ,2013W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2013,N,2012-12-26,2011-12-26,2013-09-26,2013-06-26,2013-11-26,2013-10-26,2013-12-19,2013-12-12,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-17,2013-01-01,2013-07-01,2013-09-01,2013-09-15,0,-1,-2,-10,-71,0,-1,-3,-10,-10,41533,260,79,17,3,38,5933,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-21,17-Sep-2013,Sep 17th 2013,Tue 09-17-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-15,2013-09-16,2013-09-17,Tue ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,38,2013,2013-09-15,2013-09-21,N,Wk.38 - 13,2013-09-15 00:00:00,2013-09-21 00:00:00,2013W38 ,2013W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2013,N,2012-09-17,2011-09-17,2013-06-17,2013-03-17,2013-08-17,2013-07-17,2013-09-10,2013-09-03,0,0,0,0,0,0,0,0,9,3,38,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-20,2013-01-01,2013-10-01,2013-10-01,2013-10-20,0,0,-1,-5,-38,0,0,-2,-5,-6,41566,293,20,20,1,43,5938,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-26,20-Oct-2013,Oct 20th 2013,Sun 10-20-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-20,2013-10-14,2013-10-20,Sun ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,43,2013,2013-10-20,2013-10-26,Y,Wk.43 - 13,2013-10-20 00:00:00,2013-10-26 00:00:00,2013W43 ,2013W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2013,N,2012-10-20,2011-10-20,2013-07-20,2013-04-20,2013-09-20,2013-08-20,2013-10-13,2013-10-06,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-25,2013-01-01,2013-10-01,2013-11-01,2013-11-24,0,0,0,0,-2,0,0,0,0,0,41602,329,56,25,2,48,5943,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-30,25-Nov-2013,Nov 25th 2013,Mon 11-25-13,1,1,1,1,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-11-24,2013-11-24,2013-11-25,2013-11-25,Mon ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,48,2013,2013-11-24,2013-11-30,N,Wk.48 - 13,2013-11-24 00:00:00,2013-11-30 00:00:00,2013W48 ,2013W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2013,N,2012-11-25,2011-11-25,2013-08-25,2013-05-25,2013-10-25,2013-09-25,2013-11-18,2013-11-11,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-26,2013-01-01,2013-01-01,2013-01-01,2013-01-20,0,-3,-10,-44,-305,0,-3,-11,-44,-44,41299,26,26,26,7,4,5899,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-26,26-Jan-2013,Jan 26th 2013,Sat 01-26-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2013-01-20,2013-01-21,2013-01-26,Sat ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,4,2013,2013-01-20,2013-01-26,N,Wk.04 - 13,2013-01-20 00:00:00,2013-01-26 00:00:00,2013W04 ,2013W04 ,01/20/13 - 01/26/13,01/20 - 01/26,2013,N,2012-01-26,2011-01-26,2012-10-26,2012-07-26,2012-12-26,2012-11-26,2013-01-19,2013-01-12,0,0,0,0,0,0,0,0,1,1,4,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-24,2013-01-01,2013-10-01,2013-12-01,2013-12-22,0,0,1,4,27,0,0,0,4,4,41631,358,85,24,3,52,5947,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-28,24-Dec-2013,Dec 24th 2013,Tue 12-24-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-22,2013-12-23,2013-12-24,Tue ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,52,2013,2013-12-22,2013-12-28,N,Wk.52 - 13,2013-12-22 00:00:00,2013-12-28 00:00:00,2013W52 ,2013W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2013,N,2012-12-24,2011-12-24,2013-09-24,2013-06-24,2013-11-24,2013-10-24,2013-12-17,2013-12-10,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-04,2013-01-01,2013-01-01,2013-02-01,2013-02-03,0,-3,-9,-42,-296,0,-3,-10,-42,-42,41308,35,35,4,2,6,5901,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-02-09,04-Feb-2013,Feb 4th 2013,Mon 02-04-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-01-27,2013-02-03,2013-02-04,2013-02-04,Mon ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,6,2013,2013-02-03,2013-02-09,N,Wk.06 - 13,2013-02-03 00:00:00,2013-02-09 00:00:00,2013W06 ,2013W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2013,N,2012-02-04,2011-02-04,2012-11-04,2012-08-04,2013-01-04,2012-12-04,2013-01-28,2013-01-21,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-30,2013-01-01,2013-04-01,2013-05-01,2013-05-26,0,-2,-6,-26,-181,0,-2,-6,-26,-26,41423,150,60,30,5,22,5917,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-06-01,30-May-2013,May 30th 2013,Thu 05-30-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-05-26,2013-05-26,2013-05-27,2013-05-30,Thu ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,22,2013,2013-05-26,2013-06-01,N,Wk.22 - 13,2013-05-26 00:00:00,2013-06-01 00:00:00,2013W22 ,2013W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2013,N,2012-05-30,2011-05-30,2013-02-28,2012-11-30,2013-04-30,2013-03-30,2013-05-23,2013-05-16,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-12,2013-01-01,2013-04-01,2013-04-01,2013-04-07,0,-2,-7,-33,-229,0,-2,-8,-33,-33,41375,102,12,12,6,15,5910,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-13,12-Apr-2013,Apr 12th 2013,Fri 04-12-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-07,2013-04-08,2013-04-12,Fri ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,15,2013,2013-04-07,2013-04-13,N,Wk.15 - 13,2013-04-07 00:00:00,2013-04-13 00:00:00,2013W15 ,2013W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2013,N,2012-04-12,2011-04-12,2013-01-12,2012-10-12,2013-03-12,2013-02-12,2013-04-05,2013-03-29,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-08,2013-01-01,2013-04-01,2013-04-01,2013-04-07,0,-2,-7,-33,-233,0,-2,-8,-33,-33,41371,98,8,8,2,15,5910,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-13,08-Apr-2013,Apr 8th 2013,Mon 04-08-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-07,2013-04-08,2013-04-08,Mon ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,15,2013,2013-04-07,2013-04-13,N,Wk.15 - 13,2013-04-07 00:00:00,2013-04-13 00:00:00,2013W15 ,2013W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2013,N,2012-04-08,2011-04-08,2013-01-08,2012-10-08,2013-03-08,2013-02-08,2013-04-01,2013-03-25,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-28,2013-01-01,2013-10-01,2013-12-01,2013-12-22,0,0,1,4,31,0,0,0,4,4,41635,362,89,28,7,52,5947,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-28,28-Dec-2013,Dec 28th 2013,Sat 12-28-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-22,2013-12-23,2013-12-28,Sat ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,52,2013,2013-12-22,2013-12-28,N,Wk.52 - 13,2013-12-22 00:00:00,2013-12-28 00:00:00,2013W52 ,2013W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2013,N,2012-12-28,2011-12-28,2013-09-28,2013-06-28,2013-11-28,2013-10-28,2013-12-21,2013-12-14,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-03,2013-01-01,2013-10-01,2013-12-01,2013-12-01,0,0,1,1,6,0,0,0,1,1,41610,337,64,3,3,49,5944,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-07,03-Dec-2013,Dec 3rd 2013,Tue 12-03-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-01,2013-12-02,2013-12-03,Tue ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,49,2013,2013-12-01,2013-12-07,N,Wk.49 - 13,2013-12-01 00:00:00,2013-12-07 00:00:00,2013W49 ,2013W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2013,N,2012-12-03,2011-12-03,2013-09-03,2013-06-03,2013-11-03,2013-10-03,2013-11-26,2013-11-19,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-15,2013-01-01,2013-07-01,2013-07-01,2013-07-14,0,-1,-4,-19,-135,0,-1,-5,-19,-19,41469,196,15,15,2,29,5924,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-20,15-Jul-2013,Jul 15th 2013,Mon 07-15-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-14,2013-07-15,2013-07-15,Mon ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,29,2013,2013-07-14,2013-07-20,N,Wk.29 - 13,2013-07-14 00:00:00,2013-07-20 00:00:00,2013W29 ,2013W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2013,N,2012-07-15,2011-07-15,2013-04-15,2013-01-15,2013-06-15,2013-05-15,2013-07-08,2013-07-01,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-13,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-198,0,-2,-7,-28,-28,41406,133,43,13,2,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,13-May-2013,May 13th 2013,Mon 05-13-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-13,2013-05-13,Mon ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,N,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-13,2011-05-13,2013-02-13,2012-11-13,2013-04-13,2013-03-13,2013-05-06,2013-04-29,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-30,2013-01-01,2013-01-01,2013-03-01,2013-03-24,0,-3,-8,-35,-242,0,-3,-9,-35,-35,41362,89,89,30,7,13,5908,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-30,30-Mar-2013,Mar 30th 2013,Sat 03-30-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-24,2013-03-25,2013-03-30,Sat ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,13,2013,2013-03-24,2013-03-30,N,Wk.13 - 13,2013-03-24 00:00:00,2013-03-30 00:00:00,2013W13 ,2013W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2013,N,2012-03-30,2011-03-30,2012-12-30,2012-09-30,2013-02-28,2013-01-30,2013-03-23,2013-03-16,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-09,2013-01-01,2013-10-01,2013-12-01,2013-12-08,0,0,1,2,12,0,0,0,2,2,41616,343,70,9,2,50,5945,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-14,09-Dec-2013,Dec 9th 2013,Mon 12-09-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-08,2013-12-09,2013-12-09,Mon ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,50,2013,2013-12-08,2013-12-14,N,Wk.50 - 13,2013-12-08 00:00:00,2013-12-14 00:00:00,2013W50 ,2013W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2013,N,2012-12-09,2011-12-09,2013-09-09,2013-06-09,2013-11-09,2013-10-09,2013-12-02,2013-11-25,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-25,2013-01-01,2013-01-01,2013-02-01,2013-02-24,0,-3,-9,-39,-275,0,-3,-9,-39,-39,41329,56,56,25,2,9,5904,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-03-02,25-Feb-2013,Feb 25th 2013,Mon 02-25-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-02-24,2013-02-24,2013-02-25,2013-02-25,Mon ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,9,2013,2013-02-24,2013-03-02,N,Wk.09 - 13,2013-02-24 00:00:00,2013-03-02 00:00:00,2013W09 ,2013W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2013,N,2012-02-25,2011-02-25,2012-11-25,2012-08-25,2013-01-25,2012-12-25,2013-02-18,2013-02-11,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-04,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-23,0,0,-1,-3,-3,41581,308,35,4,2,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,04-Nov-2013,Nov 4th 2013,Mon 11-04-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-11-04,2013-11-04,Mon ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,N,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-04,2011-11-04,2013-08-04,2013-05-04,2013-10-04,2013-09-04,2013-10-28,2013-10-21,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-08,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-19,0,0,-1,-3,-3,41585,312,39,8,6,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,08-Nov-2013,Nov 8th 2013,Fri 11-08-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-11-04,2013-11-08,Fri ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,N,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-08,2011-11-08,2013-08-08,2013-05-08,2013-10-08,2013-09-08,2013-11-01,2013-10-25,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-22,2013-01-01,2013-07-01,2013-08-01,2013-08-18,0,-1,-3,-14,-97,0,-1,-4,-14,-14,41507,234,53,22,5,34,5929,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-24,22-Aug-2013,Aug 22nd 2013,Thu 08-22-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-18,2013-08-19,2013-08-22,Thu ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,34,2013,2013-08-18,2013-08-24,N,Wk.34 - 13,2013-08-18 00:00:00,2013-08-24 00:00:00,2013W34 ,2013W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2013,N,2012-08-22,2011-08-22,2013-05-22,2013-02-22,2013-07-22,2013-06-22,2013-08-15,2013-08-08,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-06-18,2013-01-01,2013-04-01,2013-06-01,2013-06-16,0,-2,-5,-23,-162,0,-2,-6,-23,-23,41442,169,79,18,3,25,5920,3,6,1362,2,454,114,2013-12-31,2013-06-30,2013-06-30,2013-06-22,18-Jun-2013,Jun 18th 2013,Tue 06-18-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-05-26,2013-06-16,2013-06-17,2013-06-18,Tue ,2013M06,Jun-2013,N,2013M06 ,N,Year 2013 - Quarter 02,2013Q02 ,N,25,2013,2013-06-16,2013-06-22,N,Wk.25 - 13,2013-06-16 00:00:00,2013-06-22 00:00:00,2013W25 ,2013W25 ,06/16/13 - 06/22/13,06/16 - 06/22,2013,N,2012-06-18,2011-06-18,2013-03-18,2012-12-18,2013-05-18,2013-04-18,2013-06-11,2013-06-04,0,0,0,0,0,0,0,0,6,2,25,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-19,2013-01-01,2013-07-01,2013-07-01,2013-07-14,0,-1,-4,-19,-131,0,-1,-5,-19,-19,41473,200,19,19,6,29,5924,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-20,19-Jul-2013,Jul 19th 2013,Fri 07-19-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-14,2013-07-15,2013-07-19,Fri ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,29,2013,2013-07-14,2013-07-20,N,Wk.29 - 13,2013-07-14 00:00:00,2013-07-20 00:00:00,2013W29 ,2013W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2013,N,2012-07-19,2011-07-19,2013-04-19,2013-01-19,2013-06-19,2013-05-19,2013-07-12,2013-07-05,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-20,2013-01-01,2013-07-01,2013-08-01,2013-08-18,0,-1,-3,-14,-99,0,-1,-4,-14,-14,41505,232,51,20,3,34,5929,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-24,20-Aug-2013,Aug 20th 2013,Tue 08-20-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-18,2013-08-19,2013-08-20,Tue ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,34,2013,2013-08-18,2013-08-24,N,Wk.34 - 13,2013-08-18 00:00:00,2013-08-24 00:00:00,2013W34 ,2013W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2013,N,2012-08-20,2011-08-20,2013-05-20,2013-02-20,2013-07-20,2013-06-20,2013-08-13,2013-08-06,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-24,2013-01-01,2013-07-01,2013-08-01,2013-08-18,0,-1,-3,-14,-95,0,-1,-4,-14,-14,41509,236,55,24,7,34,5929,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-24,24-Aug-2013,Aug 24th 2013,Sat 08-24-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-18,2013-08-19,2013-08-24,Sat ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,34,2013,2013-08-18,2013-08-24,N,Wk.34 - 13,2013-08-18 00:00:00,2013-08-24 00:00:00,2013W34 ,2013W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2013,N,2012-08-24,2011-08-24,2013-05-24,2013-02-24,2013-07-24,2013-06-24,2013-08-17,2013-08-10,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-13,2013-01-01,2013-07-01,2013-07-01,2013-07-07,0,-1,-4,-20,-137,0,-1,-5,-20,-20,41467,194,13,13,7,28,5923,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-13,13-Jul-2013,Jul 13th 2013,Sat 07-13-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-07,2013-07-08,2013-07-13,Sat ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,28,2013,2013-07-07,2013-07-13,N,Wk.28 - 13,2013-07-07 00:00:00,2013-07-13 00:00:00,2013W28 ,2013W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2013,N,2012-07-13,2011-07-13,2013-04-13,2013-01-13,2013-06-13,2013-05-13,2013-07-06,2013-06-29,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-06,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-21,0,0,-1,-3,-3,41583,310,37,6,4,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,06-Nov-2013,Nov 6th 2013,Wed 11-06-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-11-04,2013-11-06,Wed ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,N,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-06,2011-11-06,2013-08-06,2013-05-06,2013-10-06,2013-09-06,2013-10-30,2013-10-23,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-07,2013-01-01,2013-10-01,2013-12-01,2013-12-01,0,0,1,1,10,0,0,0,1,1,41614,341,68,7,7,49,5944,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-07,07-Dec-2013,Dec 7th 2013,Sat 12-07-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-01,2013-12-02,2013-12-07,Sat ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,49,2013,2013-12-01,2013-12-07,N,Wk.49 - 13,2013-12-01 00:00:00,2013-12-07 00:00:00,2013W49 ,2013W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2013,N,2012-12-07,2011-12-07,2013-09-07,2013-06-07,2013-11-07,2013-10-07,2013-11-30,2013-11-23,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-04,2013-01-01,2013-07-01,2013-07-01,2013-06-30,0,-1,-4,-21,-146,0,-1,-5,-21,-21,41458,185,4,4,5,27,5922,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-06,04-Jul-2013,Jul 4th 2013,Thu 07-04-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-06-30,2013-07-01,2013-07-04,Thu ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,27,2013,2013-06-30,2013-07-06,N,Wk.27 - 13,2013-06-30 00:00:00,2013-07-06 00:00:00,2013W27 ,2013W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2013,N,2012-07-04,2011-07-04,2013-04-04,2013-01-04,2013-06-04,2013-05-04,2013-06-27,2013-06-20,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-07,2013-01-01,2013-10-01,2013-10-01,2013-10-06,0,0,-1,-7,-51,0,0,-2,-7,-7,41553,280,7,7,2,41,5936,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-12,07-Oct-2013,Oct 7th 2013,Mon 10-07-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-06,2013-10-07,2013-10-07,Mon ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,41,2013,2013-10-06,2013-10-12,N,Wk.41 - 13,2013-10-06 00:00:00,2013-10-12 00:00:00,2013W41 ,2013W41 ,10/06/13 - 10/12/13,10/06 - 10/12,2013,N,2012-10-07,2011-10-07,2013-07-07,2013-04-07,2013-09-07,2013-08-07,2013-09-30,2013-09-23,0,0,0,0,0,0,0,0,10,4,41,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-10,2013-01-01,2013-10-01,2013-11-01,2013-11-10,0,0,0,-2,-17,0,0,-1,-2,-3,41587,314,41,10,1,46,5941,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-16,10-Nov-2013,Nov 10th 2013,Sun 11-10-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-10,2013-11-04,2013-11-10,Sun ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,46,2013,2013-11-10,2013-11-16,Y,Wk.46 - 13,2013-11-10 00:00:00,2013-11-16 00:00:00,2013W46 ,2013W46 ,11/10/13 - 11/16/13,11/10 - 11/16,2013,N,2012-11-10,2011-11-10,2013-08-10,2013-05-10,2013-10-10,2013-09-10,2013-11-03,2013-10-27,0,0,0,0,0,0,0,0,11,4,46,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-07,2013-01-01,2013-07-01,2013-08-01,2013-08-04,0,-1,-3,-16,-112,0,-1,-4,-16,-16,41492,219,38,7,4,32,5927,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-10,07-Aug-2013,Aug 7th 2013,Wed 08-07-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-04,2013-08-05,2013-08-07,Wed ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,32,2013,2013-08-04,2013-08-10,N,Wk.32 - 13,2013-08-04 00:00:00,2013-08-10 00:00:00,2013W32 ,2013W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2013,N,2012-08-07,2011-08-07,2013-05-07,2013-02-07,2013-07-07,2013-06-07,2013-07-31,2013-07-24,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-25,2013-01-01,2013-07-01,2013-09-01,2013-09-22,0,-1,-2,-9,-63,0,-1,-3,-9,-9,41541,268,87,25,4,39,5934,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-28,25-Sep-2013,Sep 25th 2013,Wed 09-25-13,1,0,0,0,2013-09-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-22,2013-09-23,2013-09-25,Wed ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,39,2013,2013-09-22,2013-09-28,N,Wk.39 - 13,2013-09-22 00:00:00,2013-09-28 00:00:00,2013W39 ,2013W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2013,N,2012-09-25,2011-09-25,2013-06-25,2013-03-25,2013-08-25,2013-07-25,2013-09-18,2013-09-11,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-05,2013-01-01,2013-10-01,2013-10-01,2013-09-29,0,0,-1,-8,-53,0,0,-2,-8,-8,41551,278,5,5,7,40,5935,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-05,05-Oct-2013,Oct 5th 2013,Sat 10-05-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-09-29,2013-09-30,2013-10-05,Sat ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,40,2013,2013-09-29,2013-10-05,N,Wk.40 - 13,2013-09-29 00:00:00,2013-10-05 00:00:00,2013W40 ,2013W40 ,09/29/13 - 10/05/13,09/29 - 10/05,2013,N,2012-10-05,2011-10-05,2013-07-05,2013-04-05,2013-09-05,2013-08-05,2013-09-28,2013-09-21,0,0,0,0,0,0,0,0,10,4,40,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-02,2013-01-01,2013-07-01,2013-09-01,2013-09-01,0,-1,-2,-12,-86,0,-1,-3,-12,-12,41518,245,64,2,2,36,5931,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-07,02-Sep-2013,Sep 2nd 2013,Mon 09-02-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-01,2013-09-02,2013-09-02,Mon ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,36,2013,2013-09-01,2013-09-07,N,Wk.36 - 13,2013-09-01 00:00:00,2013-09-07 00:00:00,2013W36 ,2013W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2013,N,2012-09-02,2011-09-02,2013-06-02,2013-03-02,2013-08-02,2013-07-02,2013-08-26,2013-08-19,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-15,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-196,0,-2,-7,-28,-28,41408,135,45,15,4,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,15-May-2013,May 15th 2013,Wed 05-15-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-13,2013-05-15,Wed ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,N,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-15,2011-05-15,2013-02-15,2012-11-15,2013-04-15,2013-03-15,2013-05-08,2013-05-01,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-10,2013-01-01,2013-01-01,2013-02-01,2013-02-10,0,-3,-9,-41,-290,0,-3,-10,-41,-42,41314,41,41,10,1,7,5902,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-02-16,10-Feb-2013,Feb 10th 2013,Sun 02-10-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-01-27,2013-02-10,2013-02-04,2013-02-10,Sun ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,7,2013,2013-02-10,2013-02-16,Y,Wk.07 - 13,2013-02-10 00:00:00,2013-02-16 00:00:00,2013W07 ,2013W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2013,N,2012-02-10,2011-02-10,2012-11-10,2012-08-10,2013-01-10,2012-12-10,2013-02-03,2013-01-27,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-21,2013-01-01,2013-10-01,2013-12-01,2013-12-15,0,0,1,3,24,0,0,0,3,3,41628,355,82,21,7,51,5946,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-21,21-Dec-2013,Dec 21st 2013,Sat 12-21-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-15,2013-12-16,2013-12-21,Sat ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,51,2013,2013-12-15,2013-12-21,N,Wk.51 - 13,2013-12-15 00:00:00,2013-12-21 00:00:00,2013W51 ,2013W51 ,12/15/13 - 12/21/13,12/15 - 12/21,2013,N,2012-12-21,2011-12-21,2013-09-21,2013-06-21,2013-11-21,2013-10-21,2013-12-14,2013-12-07,0,0,0,0,0,0,0,0,12,4,51,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-22,2013-01-01,2013-10-01,2013-10-01,2013-10-20,0,0,-1,-5,-36,0,0,-2,-5,-5,41568,295,22,22,3,43,5938,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-26,22-Oct-2013,Oct 22nd 2013,Tue 10-22-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-20,2013-10-21,2013-10-22,Tue ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,43,2013,2013-10-20,2013-10-26,N,Wk.43 - 13,2013-10-20 00:00:00,2013-10-26 00:00:00,2013W43 ,2013W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2013,N,2012-10-22,2011-10-22,2013-07-22,2013-04-22,2013-09-22,2013-08-22,2013-10-15,2013-10-08,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-24,2013-01-01,2013-10-01,2013-10-01,2013-10-20,0,0,-1,-5,-34,0,0,-2,-5,-5,41570,297,24,24,5,43,5938,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-26,24-Oct-2013,Oct 24th 2013,Thu 10-24-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-20,2013-10-21,2013-10-24,Thu ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,43,2013,2013-10-20,2013-10-26,N,Wk.43 - 13,2013-10-20 00:00:00,2013-10-26 00:00:00,2013W43 ,2013W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2013,N,2012-10-24,2011-10-24,2013-07-24,2013-04-24,2013-09-24,2013-08-24,2013-10-17,2013-10-10,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-10,2013-01-01,2013-07-01,2013-07-01,2013-07-07,0,-1,-4,-20,-140,0,-1,-5,-20,-20,41464,191,10,10,4,28,5923,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-13,10-Jul-2013,Jul 10th 2013,Wed 07-10-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-07,2013-07-08,2013-07-10,Wed ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,28,2013,2013-07-07,2013-07-13,N,Wk.28 - 13,2013-07-07 00:00:00,2013-07-13 00:00:00,2013W28 ,2013W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2013,N,2012-07-10,2011-07-10,2013-04-10,2013-01-10,2013-06-10,2013-05-10,2013-07-03,2013-06-26,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-27,2013-01-01,2013-01-01,2013-02-01,2013-02-24,0,-3,-9,-39,-273,0,-3,-9,-39,-39,41331,58,58,27,4,9,5904,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-03-02,27-Feb-2013,Feb 27th 2013,Wed 02-27-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-02-24,2013-02-24,2013-02-25,2013-02-27,Wed ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,9,2013,2013-02-24,2013-03-02,N,Wk.09 - 13,2013-02-24 00:00:00,2013-03-02 00:00:00,2013W09 ,2013W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2013,N,2012-02-27,2011-02-27,2012-11-27,2012-08-27,2013-01-27,2012-12-27,2013-02-20,2013-02-13,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-02,2013-01-01,2013-01-01,2013-03-01,2013-02-24,0,-3,-8,-39,-270,0,-3,-9,-39,-39,41334,61,61,2,7,9,5904,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-02,02-Mar-2013,Mar 2nd 2013,Sat 03-02-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-02-24,2013-02-25,2013-03-02,Sat ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,9,2013,2013-02-24,2013-03-02,N,Wk.09 - 13,2013-02-24 00:00:00,2013-03-02 00:00:00,2013W09 ,2013W09 ,02/24/13 - 03/02/13,02/24 - 03/02,2013,N,2012-03-02,2011-03-02,2012-12-02,2012-09-02,2013-02-02,2013-01-02,2013-02-23,2013-02-16,0,0,0,0,0,0,0,0,3,1,9,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-02,2013-01-01,2013-10-01,2013-11-01,2013-10-27,0,0,0,-4,-25,0,0,-1,-4,-4,41579,306,33,2,7,44,5939,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-02,02-Nov-2013,Nov 2nd 2013,Sat 11-02-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-10-27,2013-10-28,2013-11-02,Sat ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,44,2013,2013-10-27,2013-11-02,N,Wk.44 - 13,2013-10-27 00:00:00,2013-11-02 00:00:00,2013W44 ,2013W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2013,N,2012-11-02,2011-11-02,2013-08-02,2013-05-02,2013-10-02,2013-09-02,2013-10-26,2013-10-19,0,0,0,0,0,0,0,0,11,4,44,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-21,2013-01-01,2013-01-01,2013-03-01,2013-03-17,0,-3,-8,-36,-251,0,-3,-9,-36,-36,41353,80,80,21,5,12,5907,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-23,21-Mar-2013,Mar 21st 2013,Thu 03-21-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-17,2013-03-18,2013-03-21,Thu ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,12,2013,2013-03-17,2013-03-23,N,Wk.12 - 13,2013-03-17 00:00:00,2013-03-23 00:00:00,2013W12 ,2013W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2013,N,2012-03-21,2011-03-21,2012-12-21,2012-09-21,2013-02-21,2013-01-21,2013-03-14,2013-03-07,0,0,0,0,0,0,0,0,3,1,12,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-03,2013-01-01,2013-04-01,2013-04-01,2013-03-31,0,-2,-7,-34,-238,0,-2,-8,-34,-34,41366,93,3,3,4,14,5909,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-06,03-Apr-2013,Apr 3rd 2013,Wed 04-03-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-03-31,2013-04-01,2013-04-03,Wed ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,14,2013,2013-03-31,2013-04-06,N,Wk.14 - 13,2013-03-31 00:00:00,2013-04-06 00:00:00,2013W14 ,2013W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2013,N,2012-04-03,2011-04-03,2013-01-03,2012-10-03,2013-03-03,2013-02-03,2013-03-27,2013-03-20,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-06,2013-01-01,2013-07-01,2013-07-01,2013-06-30,0,-1,-4,-21,-144,0,-1,-5,-21,-21,41460,187,6,6,7,27,5922,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-06,06-Jul-2013,Jul 6th 2013,Sat 07-06-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-06-30,2013-07-01,2013-07-06,Sat ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,27,2013,2013-06-30,2013-07-06,N,Wk.27 - 13,2013-06-30 00:00:00,2013-07-06 00:00:00,2013W27 ,2013W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2013,N,2012-07-06,2011-07-06,2013-04-06,2013-01-06,2013-06-06,2013-05-06,2013-06-29,2013-06-22,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-06-26,2013-01-01,2013-04-01,2013-06-01,2013-06-23,0,-2,-5,-22,-154,0,-2,-6,-22,-22,41450,177,87,26,4,26,5921,3,6,1362,2,454,114,2013-12-31,2013-06-30,2013-06-30,2013-06-29,26-Jun-2013,Jun 26th 2013,Wed 06-26-13,1,0,0,0,2013-06-21,365,91,30,7,2012-12-30,2013-03-31,2013-05-26,2013-06-23,2013-06-24,2013-06-26,Wed ,2013M06,Jun-2013,N,2013M06 ,N,Year 2013 - Quarter 02,2013Q02 ,N,26,2013,2013-06-23,2013-06-29,N,Wk.26 - 13,2013-06-23 00:00:00,2013-06-29 00:00:00,2013W26 ,2013W26 ,06/23/13 - 06/29/13,06/23 - 06/29,2013,N,2012-06-26,2011-06-26,2013-03-26,2012-12-26,2013-05-26,2013-04-26,2013-06-19,2013-06-12,0,0,0,0,0,0,0,0,6,2,26,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-26,2013-01-01,2013-07-01,2013-08-01,2013-08-25,0,-1,-3,-13,-93,0,-1,-3,-13,-13,41511,238,57,26,2,35,5930,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-31,26-Aug-2013,Aug 26th 2013,Mon 08-26-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-08-25,2013-08-25,2013-08-26,2013-08-26,Mon ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,35,2013,2013-08-25,2013-08-31,N,Wk.35 - 13,2013-08-25 00:00:00,2013-08-31 00:00:00,2013W35 ,2013W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2013,N,2012-08-26,2011-08-26,2013-05-26,2013-02-26,2013-07-26,2013-06-26,2013-08-19,2013-08-12,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-13,2013-01-01,2013-10-01,2013-12-01,2013-12-08,0,0,1,2,16,0,0,0,2,2,41620,347,74,13,6,50,5945,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-14,13-Dec-2013,Dec 13th 2013,Fri 12-13-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-08,2013-12-09,2013-12-13,Fri ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,50,2013,2013-12-08,2013-12-14,N,Wk.50 - 13,2013-12-08 00:00:00,2013-12-14 00:00:00,2013W50 ,2013W50 ,12/08/13 - 12/14/13,12/08 - 12/14,2013,N,2012-12-13,2011-12-13,2013-09-13,2013-06-13,2013-11-13,2013-10-13,2013-12-06,2013-11-29,0,0,0,0,0,0,0,0,12,4,50,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-02,2013-01-01,2013-07-01,2013-07-01,2013-06-30,0,-1,-4,-21,-148,0,-1,-5,-21,-21,41456,183,2,2,3,27,5922,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-06,02-Jul-2013,Jul 2nd 2013,Tue 07-02-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-06-30,2013-07-01,2013-07-02,Tue ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,27,2013,2013-06-30,2013-07-06,N,Wk.27 - 13,2013-06-30 00:00:00,2013-07-06 00:00:00,2013W27 ,2013W27 ,06/30/13 - 07/06/13,06/30 - 07/06,2013,N,2012-07-02,2011-07-02,2013-04-02,2013-01-02,2013-06-02,2013-05-02,2013-06-25,2013-06-18,0,0,0,0,0,0,0,0,7,3,27,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-05,2013-01-01,2013-07-01,2013-08-01,2013-08-04,0,-1,-3,-16,-114,0,-1,-4,-16,-16,41490,217,36,5,2,32,5927,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-10,05-Aug-2013,Aug 5th 2013,Mon 08-05-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-04,2013-08-05,2013-08-05,Mon ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,32,2013,2013-08-04,2013-08-10,N,Wk.32 - 13,2013-08-04 00:00:00,2013-08-10 00:00:00,2013W32 ,2013W32 ,08/04/13 - 08/10/13,08/04 - 08/10,2013,N,2012-08-05,2011-08-05,2013-05-05,2013-02-05,2013-07-05,2013-06-05,2013-07-29,2013-07-22,0,0,0,0,0,0,0,0,8,3,32,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-08,2013-01-01,2013-01-01,2013-02-01,2013-02-03,0,-3,-9,-42,-292,0,-3,-10,-42,-42,41312,39,39,8,6,6,5901,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-02-09,08-Feb-2013,Feb 8th 2013,Fri 02-08-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-01-27,2013-02-03,2013-02-04,2013-02-08,Fri ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,6,2013,2013-02-03,2013-02-09,N,Wk.06 - 13,2013-02-03 00:00:00,2013-02-09 00:00:00,2013W06 ,2013W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2013,N,2012-02-08,2011-02-08,2012-11-08,2012-08-08,2013-01-08,2012-12-08,2013-02-01,2013-01-25,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-14,2013-01-01,2013-01-01,2013-02-01,2013-02-10,0,-3,-9,-41,-286,0,-3,-10,-41,-41,41318,45,45,14,5,7,5902,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-02-16,14-Feb-2013,Feb 14th 2013,Thu 02-14-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-01-27,2013-02-10,2013-02-11,2013-02-14,Thu ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,7,2013,2013-02-10,2013-02-16,N,Wk.07 - 13,2013-02-10 00:00:00,2013-02-16 00:00:00,2013W07 ,2013W07 ,02/10/13 - 02/16/13,02/10 - 02/16,2013,N,2012-02-14,2011-02-14,2012-11-14,2012-08-14,2013-01-14,2012-12-14,2013-02-07,2013-01-31,0,0,0,0,0,0,0,0,2,1,7,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-21,2013-01-01,2013-07-01,2013-09-01,2013-09-15,0,-1,-2,-10,-67,0,-1,-3,-10,-10,41537,264,83,21,7,38,5933,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-21,21-Sep-2013,Sep 21st 2013,Sat 09-21-13,1,0,0,0,2013-09-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-15,2013-09-16,2013-09-21,Sat ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,38,2013,2013-09-15,2013-09-21,N,Wk.38 - 13,2013-09-15 00:00:00,2013-09-21 00:00:00,2013W38 ,2013W38 ,09/15/13 - 09/21/13,09/15 - 09/21,2013,N,2012-09-21,2011-09-21,2013-06-21,2013-03-21,2013-08-21,2013-07-21,2013-09-14,2013-09-07,0,0,0,0,0,0,0,0,9,3,38,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-19,2013-01-01,2013-01-01,2013-03-01,2013-03-17,0,-3,-8,-36,-253,0,-3,-9,-36,-36,41351,78,78,19,3,12,5907,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-23,19-Mar-2013,Mar 19th 2013,Tue 03-19-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-17,2013-03-18,2013-03-19,Tue ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,12,2013,2013-03-17,2013-03-23,N,Wk.12 - 13,2013-03-17 00:00:00,2013-03-23 00:00:00,2013W12 ,2013W12 ,03/17/13 - 03/23/13,03/17 - 03/23,2013,N,2012-03-19,2011-03-19,2012-12-19,2012-09-19,2013-02-19,2013-01-19,2013-03-12,2013-03-05,0,0,0,0,0,0,0,0,3,1,12,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-28,2013-01-01,2013-01-01,2013-01-01,2013-01-27,0,-3,-10,-43,-303,0,-3,-10,-43,-43,41301,28,28,28,2,5,5900,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-02-02,28-Jan-2013,Jan 28th 2013,Mon 01-28-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-01-27,2013-01-27,2013-01-28,2013-01-28,Mon ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,5,2013,2013-01-27,2013-02-02,N,Wk.05 - 13,2013-01-27 00:00:00,2013-02-02 00:00:00,2013W05 ,2013W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2013,N,2012-01-28,2011-01-28,2012-10-28,2012-07-28,2012-12-28,2012-11-28,2013-01-21,2013-01-14,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-03,2013-01-01,2013-07-01,2013-08-01,2013-07-28,0,-1,-3,-17,-116,0,-1,-4,-17,-17,41488,215,34,3,7,31,5926,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-03,03-Aug-2013,Aug 3rd 2013,Sat 08-03-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-07-28,2013-07-29,2013-08-03,Sat ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,31,2013,2013-07-28,2013-08-03,N,Wk.31 - 13,2013-07-28 00:00:00,2013-08-03 00:00:00,2013W31 ,2013W31 ,07/28/13 - 08/03/13,07/28 - 08/03,2013,N,2012-08-03,2011-08-03,2013-05-03,2013-02-03,2013-07-03,2013-06-03,2013-07-27,2013-07-20,0,0,0,0,0,0,0,0,8,3,31,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-06-14,2013-01-01,2013-04-01,2013-06-01,2013-06-09,0,-2,-5,-24,-166,0,-2,-6,-24,-24,41438,165,75,14,6,24,5919,3,6,1362,2,454,114,2013-12-31,2013-06-30,2013-06-30,2013-06-15,14-Jun-2013,Jun 14th 2013,Fri 06-14-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-05-26,2013-06-09,2013-06-10,2013-06-14,Fri ,2013M06,Jun-2013,N,2013M06 ,N,Year 2013 - Quarter 02,2013Q02 ,N,24,2013,2013-06-09,2013-06-15,N,Wk.24 - 13,2013-06-09 00:00:00,2013-06-15 00:00:00,2013W24 ,2013W24 ,06/09/13 - 06/15/13,06/09 - 06/15,2013,N,2012-06-14,2011-06-14,2013-03-14,2012-12-14,2013-05-14,2013-04-14,2013-06-07,2013-05-31,0,0,0,0,0,0,0,0,6,2,24,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-08,2013-01-01,2013-07-01,2013-07-01,2013-07-07,0,-1,-4,-20,-142,0,-1,-5,-20,-20,41462,189,8,8,2,28,5923,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-13,08-Jul-2013,Jul 8th 2013,Mon 07-08-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-07,2013-07-08,2013-07-08,Mon ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,28,2013,2013-07-07,2013-07-13,N,Wk.28 - 13,2013-07-07 00:00:00,2013-07-13 00:00:00,2013W28 ,2013W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2013,N,2012-07-08,2011-07-08,2013-04-08,2013-01-08,2013-06-08,2013-05-08,2013-07-01,2013-06-24,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-30,2013-01-01,2013-10-01,2013-11-01,2013-11-24,0,0,0,0,3,0,0,0,0,0,41607,334,61,30,7,48,5943,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-30,30-Nov-2013,Nov 30th 2013,Sat 11-30-13,0,0,0,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-11-24,2013-11-24,2013-11-25,2013-11-30,Sat ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,48,2013,2013-11-24,2013-11-30,N,Wk.48 - 13,2013-11-24 00:00:00,2013-11-30 00:00:00,2013W48 ,2013W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2013,N,2012-11-30,2011-11-30,2013-08-30,2013-05-30,2013-10-30,2013-09-30,2013-11-23,2013-11-16,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-10,2013-01-01,2013-01-01,2013-01-01,2013-01-06,0,-3,-10,-46,-321,0,-3,-11,-46,-46,41283,10,10,10,5,2,5897,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-12,10-Jan-2013,Jan 10th 2013,Thu 01-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2013-01-06,2013-01-07,2013-01-10,Thu ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,2,2013,2013-01-06,2013-01-12,N,Wk.02 - 13,2013-01-06 00:00:00,2013-01-12 00:00:00,2013W02 ,2013W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2013,N,2012-01-10,2011-01-10,2012-10-10,2012-07-10,2012-12-10,2012-11-10,2013-01-03,2012-12-27,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-18,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-193,0,-2,-7,-28,-28,41411,138,48,18,7,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,18-May-2013,May 18th 2013,Sat 05-18-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-13,2013-05-18,Sat ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,N,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-18,2011-05-18,2013-02-18,2012-11-18,2013-04-18,2013-03-18,2013-05-11,2013-05-04,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-15,2013-01-01,2013-04-01,2013-04-01,2013-04-14,0,-2,-7,-32,-226,0,-2,-8,-32,-32,41378,105,15,15,2,16,5911,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-20,15-Apr-2013,Apr 15th 2013,Mon 04-15-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-14,2013-04-15,2013-04-15,Mon ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,16,2013,2013-04-14,2013-04-20,N,Wk.16 - 13,2013-04-14 00:00:00,2013-04-20 00:00:00,2013W16 ,2013W16 ,04/14/13 - 04/20/13,04/14 - 04/20,2013,N,2012-04-15,2011-04-15,2013-01-15,2012-10-15,2013-03-15,2013-02-15,2013-04-08,2013-04-01,0,0,0,0,0,0,0,0,4,2,16,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-12,2013-01-01,2013-01-01,2013-03-01,2013-03-10,0,-3,-8,-37,-260,0,-3,-9,-37,-37,41344,71,71,12,3,11,5906,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-16,12-Mar-2013,Mar 12th 2013,Tue 03-12-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-10,2013-03-11,2013-03-12,Tue ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,11,2013,2013-03-10,2013-03-16,N,Wk.11 - 13,2013-03-10 00:00:00,2013-03-16 00:00:00,2013W11 ,2013W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2013,N,2012-03-12,2011-03-12,2012-12-12,2012-09-12,2013-02-12,2013-01-12,2013-03-05,2013-02-26,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-09,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-18,0,0,-1,-3,-3,41586,313,40,9,7,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,09-Nov-2013,Nov 9th 2013,Sat 11-09-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-11-04,2013-11-09,Sat ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,N,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-09,2011-11-09,2013-08-09,2013-05-09,2013-10-09,2013-09-09,2013-11-02,2013-10-26,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-14,2013-01-01,2013-01-01,2013-03-01,2013-03-10,0,-3,-8,-37,-258,0,-3,-9,-37,-37,41346,73,73,14,5,11,5906,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-16,14-Mar-2013,Mar 14th 2013,Thu 03-14-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-10,2013-03-11,2013-03-14,Thu ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,11,2013,2013-03-10,2013-03-16,N,Wk.11 - 13,2013-03-10 00:00:00,2013-03-16 00:00:00,2013W11 ,2013W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2013,N,2012-03-14,2011-03-14,2012-12-14,2012-09-14,2013-02-14,2013-01-14,2013-03-07,2013-02-28,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-03,2013-01-01,2013-07-01,2013-09-01,2013-09-01,0,-1,-2,-12,-85,0,-1,-3,-12,-12,41519,246,65,3,3,36,5931,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-07,03-Sep-2013,Sep 3rd 2013,Tue 09-03-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-01,2013-09-02,2013-09-03,Tue ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,36,2013,2013-09-01,2013-09-07,N,Wk.36 - 13,2013-09-01 00:00:00,2013-09-07 00:00:00,2013W36 ,2013W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2013,N,2012-09-03,2011-09-03,2013-06-03,2013-03-03,2013-08-03,2013-07-03,2013-08-27,2013-08-20,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-16,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-195,0,-2,-7,-28,-28,41409,136,46,16,5,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,16-May-2013,May 16th 2013,Thu 05-16-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-13,2013-05-16,Thu ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,N,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-16,2011-05-16,2013-02-16,2012-11-16,2013-04-16,2013-03-16,2013-05-09,2013-05-02,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-27,2013-01-01,2013-10-01,2013-10-01,2013-10-27,0,0,-1,-4,-31,0,0,-1,-4,-5,41573,300,27,27,1,44,5939,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-11-02,27-Oct-2013,Oct 27th 2013,Sun 10-27-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-10-27,2013-10-27,2013-10-21,2013-10-27,Sun ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,44,2013,2013-10-27,2013-11-02,Y,Wk.44 - 13,2013-10-27 00:00:00,2013-11-02 00:00:00,2013W44 ,2013W44 ,10/27/13 - 11/02/13,10/27 - 11/02,2013,N,2012-10-27,2011-10-27,2013-07-27,2013-04-27,2013-09-27,2013-08-27,2013-10-20,2013-10-13,0,0,0,0,1,0,0,0,11,4,44,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-21,2013-01-01,2013-07-01,2013-08-01,2013-08-18,0,-1,-3,-14,-98,0,-1,-4,-14,-14,41506,233,52,21,4,34,5929,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-24,21-Aug-2013,Aug 21st 2013,Wed 08-21-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-18,2013-08-19,2013-08-21,Wed ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,34,2013,2013-08-18,2013-08-24,N,Wk.34 - 13,2013-08-18 00:00:00,2013-08-24 00:00:00,2013W34 ,2013W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2013,N,2012-08-21,2011-08-21,2013-05-21,2013-02-21,2013-07-21,2013-06-21,2013-08-14,2013-08-07,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-25,2013-01-01,2013-10-01,2013-10-01,2013-10-20,0,0,-1,-5,-33,0,0,-2,-5,-5,41571,298,25,25,6,43,5938,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-26,25-Oct-2013,Oct 25th 2013,Fri 10-25-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-20,2013-10-21,2013-10-25,Fri ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,43,2013,2013-10-20,2013-10-26,N,Wk.43 - 13,2013-10-20 00:00:00,2013-10-26 00:00:00,2013W43 ,2013W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2013,N,2012-10-25,2011-10-25,2013-07-25,2013-04-25,2013-09-25,2013-08-25,2013-10-18,2013-10-11,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-24,2013-01-01,2013-07-01,2013-09-01,2013-09-22,0,-1,-2,-9,-64,0,-1,-3,-9,-9,41540,267,86,24,3,39,5934,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-28,24-Sep-2013,Sep 24th 2013,Tue 09-24-13,1,0,0,0,2013-09-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-22,2013-09-23,2013-09-24,Tue ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,39,2013,2013-09-22,2013-09-28,N,Wk.39 - 13,2013-09-22 00:00:00,2013-09-28 00:00:00,2013W39 ,2013W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2013,N,2012-09-24,2011-09-24,2013-06-24,2013-03-24,2013-08-24,2013-07-24,2013-09-17,2013-09-10,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-27,2013-01-01,2013-01-01,2013-01-01,2013-01-27,0,-3,-10,-43,-304,0,-3,-10,-43,-44,41300,27,27,27,1,5,5900,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-02-02,27-Jan-2013,Jan 27th 2013,Sun 01-27-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-01-27,2013-01-27,2013-01-21,2013-01-27,Sun ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,5,2013,2013-01-27,2013-02-02,Y,Wk.05 - 13,2013-01-27 00:00:00,2013-02-02 00:00:00,2013W05 ,2013W05 ,01/27/13 - 02/02/13,01/27 - 02/02,2013,N,2012-01-27,2011-01-27,2012-10-27,2012-07-27,2012-12-27,2012-11-27,2013-01-20,2013-01-13,0,0,0,0,0,0,0,0,2,1,5,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-10,2013-01-01,2013-01-01,2013-03-01,2013-03-10,0,-3,-8,-37,-262,0,-3,-9,-37,-38,41342,69,69,10,1,11,5906,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-16,10-Mar-2013,Mar 10th 2013,Sun 03-10-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-10,2013-03-04,2013-03-10,Sun ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,11,2013,2013-03-10,2013-03-16,Y,Wk.11 - 13,2013-03-10 00:00:00,2013-03-16 00:00:00,2013W11 ,2013W11 ,03/10/13 - 03/16/13,03/10 - 03/16,2013,N,2012-03-10,2011-03-10,2012-12-10,2012-09-10,2013-02-10,2013-01-10,2013-03-03,2013-02-24,0,0,0,0,0,0,0,0,3,1,11,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-14,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-197,0,-2,-7,-28,-28,41407,134,44,14,3,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,14-May-2013,May 14th 2013,Tue 05-14-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-13,2013-05-14,Tue ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,N,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-14,2011-05-14,2013-02-14,2012-11-14,2013-04-14,2013-03-14,2013-05-07,2013-04-30,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-31,2013-01-01,2013-10-01,2013-12-01,2013-12-29,0,0,1,5,34,1,1,1,5,5,41638,365,92,31,3,53,5948,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-31,31-Dec-2013,Dec 31st 2013,Tue 12-31-13,0,0,0,0,2013-12-21,365,92,31,3,2013-12-29,2013-12-29,2013-12-29,2013-12-29,2013-12-30,2013-12-31,Tue ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,1,2014,2013-12-29,2014-01-04,N,Wk.53 - 13,2013-12-29 00:00:00,2013-12-31 00:00:00,2013W53 ,2013W53 ,12/29/13 - 12/31/13,12/29 - 12/31,2013,N,2012-12-31,2011-12-31,2013-09-30,2013-06-30,2013-11-30,2013-10-31,2013-12-24,2013-12-17,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-01,2013-01-01,2013-07-01,2013-09-01,2013-09-01,0,-1,-2,-12,-87,0,-1,-3,-12,-13,41517,244,63,1,1,36,5931,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-07,01-Sep-2013,Sep 1st 2013,Sun 09-01-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-01,2013-08-26,2013-09-01,Sun ,2013M09,Sep-2013,Y,2013M09 ,Y,Year 2013 - Quarter 03,2013Q03 ,N,36,2013,2013-09-01,2013-09-07,Y,Wk.36 - 13,2013-09-01 00:00:00,2013-09-07 00:00:00,2013W36 ,2013W36 ,09/01/13 - 09/07/13,09/01 - 09/07,2013,N,2012-09-01,2011-09-01,2013-06-01,2013-03-01,2013-08-01,2013-07-01,2013-08-25,2013-08-18,0,0,0,0,0,0,0,0,9,3,36,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-13,2013-01-01,2013-04-01,2013-04-01,2013-04-07,0,-2,-7,-33,-228,0,-2,-8,-33,-33,41376,103,13,13,7,15,5910,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-13,13-Apr-2013,Apr 13th 2013,Sat 04-13-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-07,2013-04-08,2013-04-13,Sat ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,15,2013,2013-04-07,2013-04-13,N,Wk.15 - 13,2013-04-07 00:00:00,2013-04-13 00:00:00,2013W15 ,2013W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2013,N,2012-04-13,2011-04-13,2013-01-13,2012-10-13,2013-03-13,2013-02-13,2013-04-06,2013-03-30,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-09,2013-01-01,2013-01-01,2013-02-01,2013-02-03,0,-3,-9,-42,-291,0,-3,-10,-42,-42,41313,40,40,9,7,6,5901,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-02-09,09-Feb-2013,Feb 9th 2013,Sat 02-09-13,1,0,0,0,2012-12-21,365,90,28,7,2012-12-30,2012-12-30,2013-01-27,2013-02-03,2013-02-04,2013-02-09,Sat ,2013M02,Feb-2013,N,2013M02 ,N,Year 2013 - Quarter 01,2013Q01 ,N,6,2013,2013-02-03,2013-02-09,N,Wk.06 - 13,2013-02-03 00:00:00,2013-02-09 00:00:00,2013W06 ,2013W06 ,02/03/13 - 02/09/13,02/03 - 02/09,2013,N,2012-02-09,2011-02-09,2012-11-09,2012-08-09,2013-01-09,2012-12-09,2013-02-02,2013-01-26,0,0,0,0,0,0,0,0,2,1,6,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-27,2013-01-01,2013-10-01,2013-12-01,2013-12-22,0,0,1,4,30,0,0,0,4,4,41634,361,88,27,6,52,5947,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-28,27-Dec-2013,Dec 27th 2013,Fri 12-27-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-22,2013-12-23,2013-12-27,Fri ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,52,2013,2013-12-22,2013-12-28,N,Wk.52 - 13,2013-12-22 00:00:00,2013-12-28 00:00:00,2013W52 ,2013W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2013,N,2012-12-27,2011-12-27,2013-09-27,2013-06-27,2013-11-27,2013-10-27,2013-12-20,2013-12-13,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-19,2013-01-01,2013-07-01,2013-08-01,2013-08-18,0,-1,-3,-14,-100,0,-1,-4,-14,-14,41504,231,50,19,2,34,5929,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-24,19-Aug-2013,Aug 19th 2013,Mon 08-19-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-18,2013-08-19,2013-08-19,Mon ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,34,2013,2013-08-18,2013-08-24,N,Wk.34 - 13,2013-08-18 00:00:00,2013-08-24 00:00:00,2013W34 ,2013W34 ,08/18/13 - 08/24/13,08/18 - 08/24,2013,N,2012-08-19,2011-08-19,2013-05-19,2013-02-19,2013-07-19,2013-06-19,2013-08-12,2013-08-05,0,0,0,0,0,0,0,0,8,3,34,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-04-11,2013-01-01,2013-04-01,2013-04-01,2013-04-07,0,-2,-7,-33,-230,0,-2,-8,-33,-33,41374,101,11,11,5,15,5910,1,4,1360,2,454,114,2013-12-31,2013-06-30,2013-04-30,2013-04-13,11-Apr-2013,Apr 11th 2013,Thu 04-11-13,1,0,0,0,2013-03-21,365,91,30,7,2012-12-30,2013-03-31,2013-03-31,2013-04-07,2013-04-08,2013-04-11,Thu ,2013M04,Apr-2013,N,2013M04 ,N,Year 2013 - Quarter 02,2013Q02 ,N,15,2013,2013-04-07,2013-04-13,N,Wk.15 - 13,2013-04-07 00:00:00,2013-04-13 00:00:00,2013W15 ,2013W15 ,04/07/13 - 04/13/13,04/07 - 04/13,2013,N,2012-04-11,2011-04-11,2013-01-11,2012-10-11,2013-03-11,2013-02-11,2013-04-04,2013-03-28,0,0,0,0,0,0,0,0,4,2,15,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-06,2013-01-01,2013-10-01,2013-12-01,2013-12-01,0,0,1,1,9,0,0,0,1,1,41613,340,67,6,6,49,5944,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-07,06-Dec-2013,Dec 6th 2013,Fri 12-06-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-01,2013-12-02,2013-12-06,Fri ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,49,2013,2013-12-01,2013-12-07,N,Wk.49 - 13,2013-12-01 00:00:00,2013-12-07 00:00:00,2013W49 ,2013W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2013,N,2012-12-06,2011-12-06,2013-09-06,2013-06-06,2013-11-06,2013-10-06,2013-11-29,2013-11-22,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-31,2013-01-01,2013-01-01,2013-03-01,2013-03-31,0,-3,-8,-34,-241,0,-2,-8,-34,-35,41363,90,90,31,1,14,5909,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-04-06,31-Mar-2013,Mar 31st 2013,Sun 03-31-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2013-03-31,2013-03-31,2013-03-31,2013-03-25,2013-03-31,Sun ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,14,2013,2013-03-31,2013-04-06,Y,Wk.14 - 13,2013-03-31 00:00:00,2013-04-06 00:00:00,2013W14 ,2013W14 ,03/31/13 - 04/06/13,03/31 - 04/06,2013,N,2012-03-31,2011-03-31,2012-12-31,2012-09-30,2013-02-28,2013-01-31,2013-03-24,2013-03-17,0,0,0,0,0,0,0,0,4,2,14,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-22,2013-01-01,2013-07-01,2013-09-01,2013-09-22,0,-1,-2,-9,-66,0,-1,-3,-9,-10,41538,265,84,22,1,39,5934,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-28,22-Sep-2013,Sep 22nd 2013,Sun 09-22-13,1,0,0,0,2013-09-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-22,2013-09-16,2013-09-22,Sun ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,39,2013,2013-09-22,2013-09-28,Y,Wk.39 - 13,2013-09-22 00:00:00,2013-09-28 00:00:00,2013W39 ,2013W39 ,09/22/13 - 09/28/13,09/22 - 09/28,2013,N,2012-09-22,2011-09-22,2013-06-22,2013-03-22,2013-08-22,2013-07-22,2013-09-15,2013-09-08,0,0,0,0,0,0,0,0,9,3,39,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-06,2013-01-01,2013-01-01,2013-01-01,2013-01-06,0,-3,-10,-46,-325,0,-3,-11,-46,-47,41279,6,6,6,1,2,5897,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-12,06-Jan-2013,Jan 6th 2013,Sun 01-06-13,1,0,0,0,2012-12-21,365,90,31,7,2012-12-30,2012-12-30,2012-12-30,2013-01-06,2012-12-31,2013-01-06,Sun ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,2,2013,2013-01-06,2013-01-12,Y,Wk.02 - 13,2013-01-06 00:00:00,2013-01-12 00:00:00,2013W02 ,2013W02 ,01/06/13 - 01/12/13,01/06 - 01/12,2013,N,2012-01-06,2011-01-06,2012-10-06,2012-07-06,2012-12-06,2012-11-06,2012-12-30,2012-12-23,0,0,0,0,0,0,0,0,1,1,2,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-04,2013-01-01,2013-01-01,2013-01-01,2013-01-01,0,-3,-10,-47,-327,0,-3,-11,-47,-47,41277,4,4,4,6,1,5896,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-05,04-Jan-2013,Jan 4th 2013,Fri 01-04-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2013-01-04,Fri ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,1,2013,2012-12-30,2013-01-05,N,Wk.01 - 13,2013-01-01 00:00:00,2013-01-05 00:00:00,2013W01 ,2013W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2013,N,2012-01-04,2011-01-04,2012-10-04,2012-07-04,2012-12-04,2012-11-04,2012-12-28,2012-12-21,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-16,2013-01-01,2013-07-01,2013-07-01,2013-07-14,0,-1,-4,-19,-134,0,-1,-5,-19,-19,41470,197,16,16,3,29,5924,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-20,16-Jul-2013,Jul 16th 2013,Tue 07-16-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-14,2013-07-15,2013-07-16,Tue ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,29,2013,2013-07-14,2013-07-20,N,Wk.29 - 13,2013-07-14 00:00:00,2013-07-20 00:00:00,2013W29 ,2013W29 ,07/14/13 - 07/20/13,07/14 - 07/20,2013,N,2012-07-16,2011-07-16,2013-04-16,2013-01-16,2013-06-16,2013-05-16,2013-07-09,2013-07-02,0,0,0,0,0,0,0,0,7,3,29,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-03,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-24,0,0,-1,-3,-4,41580,307,34,3,1,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,03-Nov-2013,Nov 3rd 2013,Sun 11-03-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-10-28,2013-11-03,Sun ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,Y,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-03,2011-11-03,2013-08-03,2013-05-03,2013-10-03,2013-09-03,2013-10-27,2013-10-20,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-25,2013-01-01,2013-07-01,2013-08-01,2013-08-25,0,-1,-3,-13,-94,0,-1,-3,-13,-14,41510,237,56,25,1,35,5930,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-31,25-Aug-2013,Aug 25th 2013,Sun 08-25-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-08-25,2013-08-25,2013-08-19,2013-08-25,Sun ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,35,2013,2013-08-25,2013-08-31,Y,Wk.35 - 13,2013-08-25 00:00:00,2013-08-31 00:00:00,2013W35 ,2013W35 ,08/25/13 - 08/31/13,08/25 - 08/31,2013,N,2012-08-25,2011-08-25,2013-05-25,2013-02-25,2013-07-25,2013-06-25,2013-08-18,2013-08-11,0,0,0,0,0,0,0,0,9,3,35,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-24,2013-01-01,2013-10-01,2013-11-01,2013-11-24,0,0,0,0,-3,0,0,0,0,-1,41601,328,55,24,1,48,5943,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-30,24-Nov-2013,Nov 24th 2013,Sun 11-24-13,1,1,1,1,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-11-24,2013-11-24,2013-11-18,2013-11-24,Sun ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,48,2013,2013-11-24,2013-11-30,Y,Wk.48 - 13,2013-11-24 00:00:00,2013-11-30 00:00:00,2013W48 ,2013W48 ,11/24/13 - 11/30/13,11/24 - 11/30,2013,N,2012-11-24,2011-11-24,2013-08-24,2013-05-24,2013-10-24,2013-09-24,2013-11-17,2013-11-10,0,0,0,0,0,0,0,0,12,4,48,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-12,2013-01-01,2013-04-01,2013-05-01,2013-05-12,0,-2,-6,-28,-199,0,-2,-7,-28,-29,41405,132,42,12,1,20,5915,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-18,12-May-2013,May 12th 2013,Sun 05-12-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-12,2013-05-06,2013-05-12,Sun ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,20,2013,2013-05-12,2013-05-18,Y,Wk.20 - 13,2013-05-12 00:00:00,2013-05-18 00:00:00,2013W20 ,2013W20 ,05/12/13 - 05/18/13,05/12 - 05/18,2013,N,2012-05-12,2011-05-12,2013-02-12,2012-11-12,2013-04-12,2013-03-12,2013-05-05,2013-04-28,0,0,0,0,0,0,0,0,5,2,20,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-08-17,2013-01-01,2013-07-01,2013-08-01,2013-08-11,0,-1,-3,-15,-102,0,-1,-4,-15,-15,41502,229,48,17,7,33,5928,2,8,1364,3,455,114,2013-12-31,2013-09-30,2013-08-31,2013-08-17,17-Aug-2013,Aug 17th 2013,Sat 08-17-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-07-28,2013-08-11,2013-08-12,2013-08-17,Sat ,2013M08,Aug-2013,N,2013M08 ,N,Year 2013 - Quarter 03,2013Q03 ,N,33,2013,2013-08-11,2013-08-17,N,Wk.33 - 13,2013-08-11 00:00:00,2013-08-17 00:00:00,2013W33 ,2013W33 ,08/11/13 - 08/17/13,08/11 - 08/17,2013,N,2012-08-17,2011-08-17,2013-05-17,2013-02-17,2013-07-17,2013-06-17,2013-08-10,2013-08-03,0,0,0,0,0,0,0,0,8,3,33,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-21,2013-01-01,2013-10-01,2013-10-01,2013-10-20,0,0,-1,-5,-37,0,0,-2,-5,-5,41567,294,21,21,2,43,5938,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-26,21-Oct-2013,Oct 21st 2013,Mon 10-21-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-20,2013-10-21,2013-10-21,Mon ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,43,2013,2013-10-20,2013-10-26,N,Wk.43 - 13,2013-10-20 00:00:00,2013-10-26 00:00:00,2013W43 ,2013W43 ,10/20/13 - 10/26/13,10/20 - 10/26,2013,N,2012-10-21,2011-10-21,2013-07-21,2013-04-21,2013-09-21,2013-08-21,2013-10-14,2013-10-07,0,0,0,0,0,0,0,0,10,4,43,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-03-27,2013-01-01,2013-01-01,2013-03-01,2013-03-24,0,-3,-8,-35,-245,0,-3,-9,-35,-35,41359,86,86,27,4,13,5908,3,3,1359,1,453,114,2013-12-31,2013-03-31,2013-03-31,2013-03-30,27-Mar-2013,Mar 27th 2013,Wed 03-27-13,1,0,0,0,2013-03-21,365,90,31,7,2012-12-30,2012-12-30,2013-02-24,2013-03-24,2013-03-25,2013-03-27,Wed ,2013M03,Mar-2013,N,2013M03 ,N,Year 2013 - Quarter 01,2013Q01 ,N,13,2013,2013-03-24,2013-03-30,N,Wk.13 - 13,2013-03-24 00:00:00,2013-03-30 00:00:00,2013W13 ,2013W13 ,03/24/13 - 03/30/13,03/24 - 03/30,2013,N,2012-03-27,2011-03-27,2012-12-27,2012-09-27,2013-02-27,2013-01-27,2013-03-20,2013-03-13,0,0,0,0,0,0,0,0,3,1,13,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-22,2013-01-01,2013-10-01,2013-11-01,2013-11-17,0,0,0,-1,-5,0,0,-1,-1,-1,41599,326,53,22,6,47,5942,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-23,22-Nov-2013,Nov 22nd 2013,Fri 11-22-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-17,2013-11-18,2013-11-22,Fri ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,47,2013,2013-11-17,2013-11-23,N,Wk.47 - 13,2013-11-17 00:00:00,2013-11-23 00:00:00,2013W47 ,2013W47 ,11/17/13 - 11/23/13,11/17 - 11/23,2013,N,2012-11-22,2011-11-22,2013-08-22,2013-05-22,2013-10-22,2013-09-22,2013-11-15,2013-11-08,0,0,0,0,0,0,0,0,11,4,47,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-07-07,2013-01-01,2013-07-01,2013-07-01,2013-07-07,0,-1,-4,-20,-143,0,-1,-5,-20,-21,41461,188,7,7,1,28,5923,1,7,1363,3,455,114,2013-12-31,2013-09-30,2013-07-31,2013-07-13,07-Jul-2013,Jul 7th 2013,Sun 07-07-13,1,0,0,0,2013-06-21,365,92,31,7,2012-12-30,2013-06-30,2013-06-30,2013-07-07,2013-07-01,2013-07-07,Sun ,2013M07,Jul-2013,N,2013M07 ,N,Year 2013 - Quarter 03,2013Q03 ,N,28,2013,2013-07-07,2013-07-13,Y,Wk.28 - 13,2013-07-07 00:00:00,2013-07-13 00:00:00,2013W28 ,2013W28 ,07/07/13 - 07/13/13,07/07 - 07/13,2013,N,2012-07-07,2011-07-07,2013-04-07,2013-01-07,2013-06-07,2013-05-07,2013-06-30,2013-06-23,0,0,0,0,0,0,0,0,7,3,28,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-09-11,2013-01-01,2013-07-01,2013-09-01,2013-09-08,0,-1,-2,-11,-77,0,-1,-3,-11,-11,41527,254,73,11,4,37,5932,3,9,1365,3,455,114,2013-12-31,2013-09-30,2013-09-30,2013-09-14,11-Sep-2013,Sep 11th 2013,Wed 09-11-13,1,0,0,0,2013-06-21,365,92,30,7,2012-12-30,2013-06-30,2013-08-25,2013-09-08,2013-09-09,2013-09-11,Wed ,2013M09,Sep-2013,N,2013M09 ,N,Year 2013 - Quarter 03,2013Q03 ,N,37,2013,2013-09-08,2013-09-14,N,Wk.37 - 13,2013-09-08 00:00:00,2013-09-14 00:00:00,2013W37 ,2013W37 ,09/08/13 - 09/14/13,09/08 - 09/14,2013,N,2012-09-11,2011-09-11,2013-06-11,2013-03-11,2013-08-11,2013-07-11,2013-09-04,2013-08-28,0,0,0,0,0,0,0,0,9,3,37,3,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-10-19,2013-01-01,2013-10-01,2013-10-01,2013-10-13,0,0,-1,-6,-39,0,0,-2,-6,-6,41565,292,19,19,7,42,5937,1,10,1366,4,456,114,2013-12-31,2013-12-31,2013-10-31,2013-10-19,19-Oct-2013,Oct 19th 2013,Sat 10-19-13,1,1,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-09-29,2013-10-13,2013-10-14,2013-10-19,Sat ,2013M10,Oct-2013,N,2013M10 ,N,Year 2013 - Quarter 04,2013Q04 ,N,42,2013,2013-10-13,2013-10-19,N,Wk.42 - 13,2013-10-13 00:00:00,2013-10-19 00:00:00,2013W42 ,2013W42 ,10/13/13 - 10/19/13,10/13 - 10/19,2013,N,2012-10-19,2011-10-19,2013-07-19,2013-04-19,2013-09-19,2013-08-19,2013-10-12,2013-10-05,0,0,0,0,0,0,0,0,10,4,42,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-23,2013-01-01,2013-10-01,2013-12-01,2013-12-22,0,0,1,4,26,0,0,0,4,4,41630,357,84,23,2,52,5947,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-28,23-Dec-2013,Dec 23rd 2013,Mon 12-23-13,0,0,0,0,2013-12-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-22,2013-12-23,2013-12-23,Mon ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,52,2013,2013-12-22,2013-12-28,N,Wk.52 - 13,2013-12-22 00:00:00,2013-12-28 00:00:00,2013W52 ,2013W52 ,12/22/13 - 12/28/13,12/22 - 12/28,2013,N,2012-12-23,2011-12-23,2013-09-23,2013-06-23,2013-11-23,2013-10-23,2013-12-16,2013-12-09,0,0,0,0,0,0,0,0,12,4,52,1,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-24,2013-01-01,2013-04-01,2013-05-01,2013-05-19,0,-2,-6,-27,-187,0,-2,-7,-27,-27,41417,144,54,24,6,21,5916,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-05-25,24-May-2013,May 24th 2013,Fri 05-24-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-04-28,2013-05-19,2013-05-20,2013-05-24,Fri ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,21,2013,2013-05-19,2013-05-25,N,Wk.21 - 13,2013-05-19 00:00:00,2013-05-25 00:00:00,2013W21 ,2013W21 ,05/19/13 - 05/25/13,05/19 - 05/25,2013,N,2012-05-24,2011-05-24,2013-02-24,2012-11-24,2013-04-24,2013-03-24,2013-05-17,2013-05-10,0,0,0,0,0,0,0,0,5,2,21,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-11-05,2013-01-01,2013-10-01,2013-11-01,2013-11-03,0,0,0,-3,-22,0,0,-1,-3,-3,41582,309,36,5,3,45,5940,2,11,1367,4,456,114,2013-12-31,2013-12-31,2013-11-30,2013-11-09,05-Nov-2013,Nov 5th 2013,Tue 11-05-13,1,1,1,0,2013-09-21,365,92,30,7,2012-12-30,2013-09-29,2013-10-27,2013-11-03,2013-11-04,2013-11-05,Tue ,2013M11,Nov-2013,N,2013M11 ,N,Year 2013 - Quarter 04,2013Q04 ,N,45,2013,2013-11-03,2013-11-09,N,Wk.45 - 13,2013-11-03 00:00:00,2013-11-09 00:00:00,2013W45 ,2013W45 ,11/03/13 - 11/09/13,11/03 - 11/09,2013,N,2012-11-05,2011-11-05,2013-08-05,2013-05-05,2013-10-05,2013-09-05,2013-10-29,2013-10-22,0,0,0,0,0,0,0,0,11,4,45,4,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-05-29,2013-01-01,2013-04-01,2013-05-01,2013-05-26,0,-2,-6,-26,-182,0,-2,-6,-26,-26,41422,149,59,29,4,22,5917,2,5,1361,2,454,114,2013-12-31,2013-06-30,2013-05-31,2013-06-01,29-May-2013,May 29th 2013,Wed 05-29-13,1,0,0,0,2013-03-21,365,91,31,7,2012-12-30,2013-03-31,2013-05-26,2013-05-26,2013-05-27,2013-05-29,Wed ,2013M05,May-2013,N,2013M05 ,N,Year 2013 - Quarter 02,2013Q02 ,N,22,2013,2013-05-26,2013-06-01,N,Wk.22 - 13,2013-05-26 00:00:00,2013-06-01 00:00:00,2013W22 ,2013W22 ,05/26/13 - 06/01/13,05/26 - 06/01,2013,N,2012-05-29,2011-05-29,2013-02-28,2012-11-29,2013-04-29,2013-03-29,2013-05-22,2013-05-15,0,0,0,0,0,0,0,0,6,2,22,2,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-01-02,2013-01-01,2013-01-01,2013-01-01,2013-01-01,0,-3,-10,-47,-329,0,-3,-11,-47,-47,41275,2,2,2,4,1,5896,1,1,1357,1,453,114,2013-12-31,2013-03-31,2013-01-31,2013-01-05,02-Jan-2013,Jan 2nd 2013,Wed 01-02-13,1,0,0,0,2012-12-21,365,90,31,5,2012-12-30,2012-12-30,2012-12-30,2012-12-30,2012-12-31,2013-01-02,Wed ,2013M01,Jan-2013,N,2013M01 ,N,Year 2013 - Quarter 01,2013Q01 ,N,1,2013,2012-12-30,2013-01-05,N,Wk.01 - 13,2013-01-01 00:00:00,2013-01-05 00:00:00,2013W01 ,2013W01 ,01/01/13 - 01/05/13,01/01 - 01/05,2013,N,2012-01-02,2011-01-02,2012-10-02,2012-07-02,2012-12-02,2012-11-02,2012-12-26,2012-12-19,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-12-02,2013-01-01,2013-10-01,2013-12-01,2013-12-01,0,0,1,1,5,0,0,0,1,1,41609,336,63,2,2,49,5944,3,12,1368,4,456,114,2013-12-31,2013-12-31,2013-12-31,2013-12-07,02-Dec-2013,Dec 2nd 2013,Mon 12-02-13,0,0,0,0,2013-09-21,365,92,31,7,2012-12-30,2013-09-29,2013-11-24,2013-12-01,2013-12-02,2013-12-02,Mon ,2013M12,Dec-2013,N,2013M12 ,N,Year 2013 - Quarter 04,2013Q04 ,N,49,2013,2013-12-01,2013-12-07,N,Wk.49 - 13,2013-12-01 00:00:00,2013-12-07 00:00:00,2013W49 ,2013W49 ,12/01/13 - 12/07/13,12/01 - 12/07,2013,N,2012-12-02,2011-12-02,2013-09-02,2013-06-02,2013-11-02,2013-10-02,2013-11-25,2013-11-18,0,0,0,0,0,0,0,0,12,4,49,4,0,1,0,2005-09-07,DW_BATCH ,2013-11-27 00:16:56,DW_BATCH
-2013-02-24,2013-01-01,2013-01-01,2013-02-01,2013-02-24,0,-3,-9,-39,-276,0,-3,-9,-39,-40,41328,55,55,24,1,9,5904,2,2,1358,1,453,114,2013-12-31,2013-03-31,2013-02-28,2013-03-02,24-Feb-
<TRUNCATED>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/expected_result/flat_item/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/expected_result/flat_item/part-r-00000 b/job/src/test/resources/expected_result/flat_item/part-r-00000
deleted file mode 100644
index 452085d..0000000
Binary files a/job/src/test/resources/expected_result/flat_item/part-r-00000 and /dev/null differ
[35/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
deleted file mode 100644
index 1a3c9cb..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testEncodeWithoutSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
- // CubeSegment seg = cube.getTheOnlySegment();
- CubeDesc cubeDesc = cube.getDescriptor();
- // String data =
- // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
- byte[][] data = new byte[8][];
- data[0] = Bytes.toBytes("2012-12-15");
- data[1] = Bytes.toBytes("11848");
- data[2] = Bytes.toBytes("Health & Beauty");
- data[3] = Bytes.toBytes("Fragrances");
- data[4] = Bytes.toBytes("Women");
- data[5] = Bytes.toBytes("FP-GTC");
- data[6] = Bytes.toBytes("0");
- data[7] = Bytes.toBytes("15");
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
-
- byte[] encodedKey = rowKeyEncoder.encode(data);
- assertEquals(30, encodedKey.length);
- byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, 8);
- byte[] rest = Arrays.copyOfRange(encodedKey, 8, encodedKey.length);
- assertEquals(255, Bytes.toLong(cuboidId));
- assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
- }
-
- @Test
- public void testEncodeWithSlr() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
- // CubeSegment seg = cube.getTheOnlySegment();
- CubeDesc cubeDesc = cube.getDescriptor();
- // String data =
- // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
- byte[][] data = new byte[9][];
- data[0] = Bytes.toBytes("123456789");
- data[1] = Bytes.toBytes("2012-12-15");
- data[2] = Bytes.toBytes("11848");
- data[3] = Bytes.toBytes("Health & Beauty");
- data[4] = Bytes.toBytes("Fragrances");
- data[5] = Bytes.toBytes("Women");
- data[6] = Bytes.toBytes("FP-GTC");
- data[7] = Bytes.toBytes("0");
- data[8] = Bytes.toBytes("15");
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
-
- byte[] encodedKey = rowKeyEncoder.encode(data);
- assertEquals(48, encodedKey.length);
- byte[] sellerId = Arrays.copyOfRange(encodedKey, 8, 26);
- byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, 8);
- byte[] rest = Arrays.copyOfRange(encodedKey, 26, encodedKey.length);
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
- }
-
- @Test
- public void testEncodeWithSlr2() throws Exception {
- CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
- // CubeSegment seg = cube.getTheOnlySegment();
- CubeDesc cubeDesc = cube.getDescriptor();
- // String data =
- // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
- byte[][] data = new byte[9][];
- data[0] = Bytes.toBytes("123456789");
- data[1] = null;
- data[2] = null;
- data[3] = null;
- data[4] = null;
- data[5] = null;
- data[6] = null;
- data[7] = null;
- data[8] = null;
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);
-
- byte[] encodedKey = rowKeyEncoder.encode(data);
- assertEquals(48, encodedKey.length);
- byte[] sellerId = Arrays.copyOfRange(encodedKey, 8, 26);
- byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, 8);
- byte[] rest = Arrays.copyOfRange(encodedKey, 26, encodedKey.length);
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/kv/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/kv/RowValueDecoderTest.java b/cube/src/test/java/org/apache/kylin/cube/kv/RowValueDecoderTest.java
deleted file mode 100644
index a00afef..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/kv/RowValueDecoderTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class RowValueDecoderTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testDecode() throws Exception {
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
-
- MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
- BigDecimal sum = new BigDecimal("333.1234567");
- BigDecimal min = new BigDecimal("333.1111111");
- BigDecimal max = new BigDecimal("333.1999999");
- LongWritable count = new LongWritable(2);
- LongWritable item_count = new LongWritable(99999);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- codec.encode(new Object[] { sum, min, max, count, item_count }, buf);
-
- buf.flip();
- byte[] valueBytes = new byte[buf.limit()];
- System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());
-
- RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc aggrFunc = measure.getFunction();
- int index = hbaseCol.findMeasureIndex(aggrFunc);
- rowValueDecoder.setIndex(index);
- }
-
- rowValueDecoder.decode(valueBytes);
- List<String> measureNames = rowValueDecoder.getNames();
- Object[] measureValues = rowValueDecoder.getValues();
-
- assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__, ITEM_COUNT]", measureNames.toString());
- assertEquals("[333.1235, 333.1111, 333.2000, 2, 99999]", Arrays.toString(measureValues));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java b/cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
deleted file mode 100644
index 5b708bf..0000000
--- a/cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.project;
-
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author xduo
- */
-public class ProjectManagerTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test(expected = IllegalStateException.class)
- public void testDropNonemptyProject() throws IOException {
- ProjectManager.getInstance(getTestConfig()).dropProject("DEFAULT");
- }
-
- @Test(expected = IllegalStateException.class)
- public void testDropNonexistProject() throws IOException {
- ProjectManager.getInstance(getTestConfig()).dropProject("DEFAULT???");
- }
-
- @Test
- public void testNewProject() throws Exception {
- ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
-
- int originalProjectCount = prjMgr.listAllProjects().size();
- int originalCubeCount = cubeMgr.listAllCubes().size();
- int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
- ResourceStore store = getStore();
-
- // clean legacy in case last run failed
- store.deleteResource("/cube/cube_in_alien_project.json");
-
- CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
- assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
- ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
- Set<IRealization> realizations = proMgr.listAllRealizations("alien");
- assertTrue(realizations.contains(createdCube));
-
- System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
-
- assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
- assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
- assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
-
- prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
- assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
- assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
- assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("default").contains(createdCube));
-
- prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
- assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
- assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
- assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("alien").contains(createdCube));
-
- CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);
-
- assertTrue(createdCube == droppedCube);
- assertNull(cubeMgr.getCube("cube_in_alien_project"));
- assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
- assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
-
- prjMgr.dropProject("alien");
- assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
- }
-
- @Test
- public void testExistingProject() throws Exception {
- ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
-
- int originalProjectCount = prjMgr.listAllProjects().size();
- int originalCubeCount = cubeMgr.listAllCubes().size();
- ResourceStore store = getStore();
-
- // clean legacy in case last run failed
- store.deleteResource("/cube/new_cube_in_default.json");
-
- CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
- CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
- assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));
-
- System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
-
- assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
- assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
-
- CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);
-
- assertTrue(createdCube == droppedCube);
- assertNull(cubeMgr.getCube("new_cube_in_default"));
- assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
- assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
- }
-
- @Test
- public void testProjectsDrop() throws IOException {
- ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
-
- CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
- assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
- assertTrue(prjMgr.listAllRealizations("default").contains(cube));
-
- cubeMgr.dropCube(cube.getName(), true);
-
- assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
- assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
- }
-
- @Test
- public void testProjectsLoadAfterProjectChange() throws IOException {
- ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
-
- CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
- assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
-
- prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());
-
- assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
-
- prjMgr.moveRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester");
-
- assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
deleted file mode 100644
index f797cda..0000000
--- a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata;
-
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test the data model upgrade
- * @author shaoshi
- *
- */
-@Ignore("Not needed, the migrate and test has been moved to CubeMetadataUpgrade.java")
-public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata("../meta_upgrade/");
- }
-
- @After
- public void after() throws Exception {
- //this.cleanupTestMetadata();
- }
-
- @Test
- public void testCubeDescUpgrade() throws Exception {
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
- CubeDescManager.getInstance(config);
- // CubeManager cubeManager = CubeManager.getInstance(config);
- // ProjectManager.getInstance(config);
-
- }
-
- //@Test
- public void testTableDescUpgrade() throws Exception {
-
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- TableDesc fact = metaMgr.getTableDesc("default.test_kylin_fact");
-
- @SuppressWarnings("deprecation")
- String oldResLocation = fact.getResourcePathV1();
- String newResLocation = fact.getResourcePath();
-
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- Assert.assertTrue(store.exists(newResLocation));
- Assert.assertTrue(!store.exists(oldResLocation));
-
- String oldExdResLocation = TableDesc.concatExdResourcePath("test_kylin_fact".toUpperCase());
- String newExdResLocation = TableDesc.concatExdResourcePath("default.test_kylin_fact".toUpperCase());
-
- Assert.assertTrue(store.exists(newExdResLocation));
- Assert.assertTrue(!store.exists(oldExdResLocation));
-
- }
-
- private void checkCubeDesc(String descName) {
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
- CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
- Assert.assertNotNull(cubedesc1);
- DataModelDesc model = cubedesc1.getModel();
- Assert.assertNotNull(model);
- Assert.assertTrue(model.getLookups().length > 0);
-
- List<DimensionDesc> dims = cubedesc1.getDimensions();
-
- Assert.assertTrue(dims.size() > 0);
-
- for (DimensionDesc dim : dims) {
- Assert.assertTrue(dim.getColumn().length > 0);
- }
-
- Assert.assertTrue(cubedesc1.getMeasures().size() > 0);
-
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);
-
- Assert.assertTrue(cubes.size() > 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
deleted file mode 100644
index 7eebd72..0000000
--- a/cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.measure;
-
-import static org.junit.Assert.assertTrue;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.measure.bitmap.BitmapCounter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class MeasureCodecTest {
-
- @Test
- public void basicTest() {
- MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16"), measure("bitmap") };
- MeasureCodec codec = new MeasureCodec(descs);
-
- DoubleWritable d = new DoubleWritable(1.0);
- LongWritable l = new LongWritable(2);
- BigDecimal b = new BigDecimal("333.1234");
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16);
- hllc.add("1234567");
- hllc.add("abcdefg");
- HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16);
- hllc.add("1234567");
- hllc.add("abcdefg");
- BitmapCounter bitmap = new BitmapCounter();
- bitmap.add(123);
- bitmap.add(45678);
- bitmap.add(Long.MAX_VALUE - 10);
- Object values[] = new Object[] { d, l, b, hllc, hllc2, bitmap };
-
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- codec.encode(values, buf);
- buf.flip();
- System.out.println("size: " + buf.limit());
-
- Object copy[] = new Object[values.length];
- codec.decode(buf, copy);
-
- assertTrue(Arrays.equals(values, copy));
- }
-
- private MeasureDesc measure(String returnType) {
- MeasureDesc desc = new MeasureDesc();
- FunctionDesc func = new FunctionDesc();
- func.setReturnType(returnType);
- desc.setFunction(func);
- return desc;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/resources/data/TEST1_desc.json
----------------------------------------------------------------------
diff --git a/cube/src/test/resources/data/TEST1_desc.json b/cube/src/test/resources/data/TEST1_desc.json
deleted file mode 100644
index 0373496..0000000
--- a/cube/src/test/resources/data/TEST1_desc.json
+++ /dev/null
@@ -1,188 +0,0 @@
-{
- "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
- "last_modified" : 1401429176099,
- "name" : "TEST1_desc",
- "model_name": "TEST1_model_desc",
- "fact_table" : "TEST_KYLIN_FACT",
- "dimensions" : [ {
- "id" : 1,
- "name" : "CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- },
- "hierarchy" : null,
- "table" : "TEST_CAL_DT",
- "column" : ["TEST_CAL_DT.CAL_DT"],
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "id" : 2,
- "name" : "CATEGORY",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- },
- "hierarchy" : true,
- "table" : "TEST_CATEGORY_GROUPINGS",
- "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"],
- "datatype" : null,
- "derived" : null
- }, {
- "id" : 3,
- "name" : "LSTG_FORMAT_NAME",
- "join" : null,
- "hierarchy" : null,
- "table" : "TEST_KYLIN_FACT",
- "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"],
- "datatype" : "string",
- "derived" : null
- }, {
- "id" : 4,
- "name" : "SITE_ID",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- },
- "hierarchy" : null,
- "table" : "TEST_SITES",
- "column" : ["TEST_SITES.SITE_ID"],
- "datatype" : "string",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "id" : 5,
- "name" : "SELLER_TYPE_CD",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- },
- "hierarchy" : null,
- "table" : "TEST_SELLER_TYPE_DIM",
- "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"],
- "datatype" : "string",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "id" : 1,
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 2,
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 3,
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 4,
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1"
- },
- "returntype" : "long"
- }
- }, {
- "id" : 5,
- "name" : "SELLER_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID"
- },
- "returntype" : "hllc10"
- }
- }, {
- "id" : 6,
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
- },
- "returntype" : "hllc10"
- }
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "CAL_DT",
- "length" : 0,
- "dictionary" : "date(yyyy-mm-dd)",
- "mandatory" : true
- }, {
- "column" : "META_CATEG_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "CATEG_LVL2_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "CATEG_LVL3_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "LSTG_FORMAT_NAME",
- "length" : 12,
- "dictionary" : null,
- "mandatory" : false
- }, {
- "column" : "SITE_ID",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "SELLER_TYPE_CD",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- } ],
- "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ]
- },
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ]
- } ]
- } ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/resources/data/TEST2_desc.json
----------------------------------------------------------------------
diff --git a/cube/src/test/resources/data/TEST2_desc.json b/cube/src/test/resources/data/TEST2_desc.json
deleted file mode 100644
index de6bfe0..0000000
--- a/cube/src/test/resources/data/TEST2_desc.json
+++ /dev/null
@@ -1,173 +0,0 @@
-{
- "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
- "last_modified" : 1401429176099,
- "model_name": "TEST2_model_desc",
- "name" : "TEST2_desc",
- "dimensions" : [ {
- "name" : "CAL_DT",
- "hierarchy" : null,
- "column" : ["TEST_CAL_DT.CAL_DT"],
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "id" : 2,
- "name" : "CATEGORY",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- },
- "hierarchy" : true,
- "column" : ["TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"],
- "derived" : null
- }, {
- "id" : 3,
- "name" : "LSTG_FORMAT_NAME",
- "hierarchy" : null,
- "table" : "TEST_KYLIN_FACT",
- "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"],
- "derived" : null
- }, {
- "id" : 4,
- "name" : "SITE_ID",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- },
- "hierarchy" : null,
- "column" : ["TEST_SITES.SITE_ID"],
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "id" : 5,
- "name" : "SELLER_TYPE_CD",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- },
- "hierarchy" : null,
- "table" : "TEST_SELLER_TYPE_DIM",
- "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"],
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "id" : 1,
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 2,
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 3,
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE"
- },
- "returntype" : "decimal"
- }
- }, {
- "id" : 4,
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1"
- },
- "returntype" : "long"
- }
- }, {
- "id" : 5,
- "name" : "SELLER_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID"
- },
- "returntype" : "hllc10"
- }
- }, {
- "id" : 6,
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
- },
- "returntype" : "hllc10"
- }
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "CAL_DT",
- "length" : 0,
- "dictionary" : "date(yyyy-mm-dd)",
- "mandatory" : true
- }, {
- "column" : "META_CATEG_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "CATEG_LVL2_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "CATEG_LVL3_NAME",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "LSTG_FORMAT_NAME",
- "length" : 12,
- "dictionary" : null,
- "mandatory" : false
- }, {
- "column" : "SITE_ID",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- }, {
- "column" : "SELLER_TYPE_CD",
- "length" : 0,
- "dictionary" : "string",
- "mandatory" : false
- } ],
- "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD", "CATEG_LVL2_NAME" ] ]
- },
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ]
- } ]
- } ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/test/resources/data/TEST3_desc.json
----------------------------------------------------------------------
diff --git a/cube/src/test/resources/data/TEST3_desc.json b/cube/src/test/resources/data/TEST3_desc.json
deleted file mode 100644
index 54f0cf9..0000000
--- a/cube/src/test/resources/data/TEST3_desc.json
+++ /dev/null
@@ -1,268 +0,0 @@
-{
- "uuid": "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b",
- "last_modified": 1401429176099,
- "name": "TEST1_desc",
- "model_name": "TEST1_model_desc",
- "dimensions": [
- {
- "id": 1,
- "name": "CAL_DT",
- "join": {
- "type": "inner",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- },
- "hierarchy": null,
- "table": "TEST_CAL_DT",
- "column": [
- "TEST_CAL_DT.CAL_DT"
- ],
- "datatype": "date",
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "id": 2,
- "name": "CATEGORY",
- "join": {
- "type": "inner",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- },
- "hierarchy": true,
- "table": "TEST_CATEGORY_GROUPINGS",
- "column": [
- "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
- "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"
- ],
- "derived": null
- },
- {
- "id": 3,
- "name": "LSTG_FORMAT_NAME",
- "join": null,
- "hierarchy": null,
- "table": "TEST_KYLIN_FACT",
- "column": [
- "TEST_KYLIN_FACT.LSTG_FORMAT_NAME"
- ],
- "datatype": "string",
- "derived": null
- },
- {
- "id": 4,
- "name": "SITE_ID",
- "join": {
- "type": "inner",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- },
- "hierarchy": null,
- "table": "TEST_SITES",
- "column": [
- "TEST_SITES.SITE_ID"
- ],
- "datatype": "string",
- "derived": [
- "SITE_NAME",
- "CRE_USER"
- ]
- },
- {
- "id": 5,
- "name": "SELLER_TYPE_CD",
- "join": {
- "type": "inner",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- },
- "hierarchy": null,
- "table": "TEST_SELLER_TYPE_DIM",
- "column": [
- "TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"
- ],
- "datatype": "string",
- "derived": [
- "SELLER_TYPE_DESC"
- ]
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal"
- }
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal"
- }
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal"
- }
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "long"
- }
- },
- {
- "id": 5,
- "name": "SELLER_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "SELLER_ID"
- },
- "returntype": "hllc10"
- }
- },
- {
- "id": 6,
- "name": "SELLER_FORMAT_CNT",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "LSTG_FORMAT_NAME",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "hllc10"
- }
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "CAL_DT",
- "length": 10,
- "dictionary": "date(yyyy-mm-dd)",
- "mandatory": false
- },
- {
- "column": "META_CATEG_NAME",
- "length": 0,
- "dictionary": "string",
- "mandatory": false
- },
- {
- "column": "CATEG_LVL2_NAME",
- "length": 0,
- "dictionary": "string",
- "mandatory": false
- },
- {
- "column": "CATEG_LVL3_NAME",
- "length": 0,
- "dictionary": "string",
- "mandatory": false
- },
- {
- "column": "LSTG_FORMAT_NAME",
- "length": 12,
- "dictionary": null,
- "mandatory": false
- },
- {
- "column": "SITE_ID",
- "length": 0,
- "dictionary": "string",
- "mandatory": false
- },
- {
- "column": "SELLER_TYPE_CD",
- "length": 0,
- "dictionary": "string",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "META_CATEG_NAME",
- "CATEG_LVL3_NAME",
- "CATEG_LVL2_NAME",
- "CAL_DT"
- ],
- [
- "LSTG_FORMAT_NAME",
- "SITE_ID",
- "SELLER_TYPE_CD"
- ]
- ]
- },
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "GMV_SUM",
- "GMV_MIN",
- "GMV_MAX",
- "TRANS_CNT",
- "SELLER_CNT",
- "SELLER_FORMAT_CNT"
- ]
- }
- ]
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/deploy/server.xml
----------------------------------------------------------------------
diff --git a/deploy/server.xml b/deploy/server.xml
deleted file mode 100644
index 256b400..0000000
--- a/deploy/server.xml
+++ /dev/null
@@ -1,148 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="9005" shutdown="SHUTDOWN">
- <!-- Security listener. Documentation at /docs/config/listeners.html
- <Listener className="org.apache.catalina.security.SecurityListener" />
- -->
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener" />
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
- <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
- <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml" />
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
-
- <!-- A "Connector" represents an endpoint by which requests are received
- and responses are returned. Documentation at :
- Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
- Java AJP Connector: /docs/config/ajp.html
- APR (HTTP/AJP) Connector: /docs/apr.html
- Define a non-SSL HTTP/1.1 Connector on port 8080
- -->
- <Connector port="7070" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="9443"
- compression="on"
- compressionMinSize="2048"
- noCompressionUserAgents="gozilla,traviata"
- compressableMimeType="text/html,text/xml,text/javascript,application/javascript,application/json,text/css,text/plain"
- />
- <!-- A "Connector" using the shared thread pool-->
- <!--
- <Connector executor="tomcatThreadPool"
- port="8080" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="8443" />
- -->
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the BIO implementation that requires the JSSE
- style configuration. When using the APR/native implementation, the
- OpenSSL style configuration is required as described in the APR/native
- documentation -->
- <!--
- <Connector port="8443" protocol="org.apache.coyote.http11.Http11Protocol"
- maxThreads="150" SSLEnabled="true" scheme="https" secure="true"
- clientAuth="false" sslProtocol="TLS" />
- -->
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
- <Connector port="9009" protocol="AJP/1.3" redirectPort="9443" />
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- Use the LockOutRealm to prevent attempts to guess user passwords
- via a brute-force attack -->
- <Realm className="org.apache.catalina.realm.LockOutRealm">
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
- </Realm>
-
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html
- Note: The pattern used is equivalent to using pattern="common" -->
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt"
- pattern="%h %l %u %t "%r" %s %b" />
-
- </Host>
- </Engine>
- </Service>
-</Server>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.core.resources.prefs b/dictionary/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 04cfa2c..0000000
--- a/dictionary/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.jdt.core.prefs b/dictionary/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a903301..0000000
--- a/dictionary/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.jdt.ui.prefs b/dictionary/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index dece0e6..0000000
--- a/dictionary/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
[21/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv b/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
deleted file mode 100644
index 07756c6..0000000
--- a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
+++ /dev/null
@@ -1,402 +0,0 @@
-Auction,0,12,48027,0,10,Coins,2013-07-16 08:23:09,USER_Y,Coins & Paper Money,Paper Money: World,Asia,0,Ebay,USER_S,12,Merchant,2013-03-31,2013-03-31,10000000157,2013-03-31,48027,10000001,184.21,1
-Others,0,5,164261,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Earrings,0,Ebay,USER_S,5,Occasional,2013-11-12,2013-11-10,10000000158,2013-11-12,164261,10000002,172.03,1
-Auction,15,14,82494,15,3,Books,2010-09-13 07:05:54,USER_Y,BookMagazines,NULL,Comic Books,15,Australia,USER_S,14,Regular,2013-04-06,2013-03-31,10000000161,2013-04-06,82494,10000003,66.6,1
-Auction,15,12,66767,15,16,Home & Garden,2011-06-14 07:48:40,USER_Y,Home & Garden,NULL,Dogs,15,Australia,USER_S,12,Merchant,2013-05-17,2013-05-12,10000000137,2013-05-17,66767,10000004,92.98,1
-FP-GTC,0,5,152801,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,NULL,Earrings,0,Ebay,USER_S,5,Occasional,2013-05-20,2013-05-19,10000000147,2013-05-20,152801,10000005,132.33,1
-FP-GTC,0,13,43398,0,36,Home & Living,2013-02-20 23:11:43,USER_Y,Home & Garden,NULL,Cheese & Crackers,0,Ebay,USER_S,13,Entrepreneur,2013-06-16,2013-06-16,10000000155,2013-06-16,43398,10000006,7.12,1
-Auction,0,14,95173,0,37,Health & Beauty,2008-06-03 13:36:00,USER_Y,Health & Beauty,Bath & Body,Bath Sets & Kits,0,Ebay,USER_S,14,Regular,2013-06-14,2013-06-09,10000000170,2013-06-14,95173,10000007,204.28,1
-Auction,15,13,158666,15,28,Toys,2012-09-11 20:27:22,USER_Y,ToyHobbies,Action Figures,Anime & Manga,15,Australia,USER_S,13,Entrepreneur,2013-03-22,2013-03-17,10000000173,2013-03-22,158666,10000008,35.72,1
-Auction,0,12,12688,0,57,Sothebys,2008-06-03 13:36:00,USER_Y,eBay Premier,Books & Manuscripts,Books: Other,0,Ebay,USER_S,12,Merchant,2013-03-10,2013-03-10,10000000178,2013-03-10,12688,10000009,4.13,1
-FP-GTC,15,5,103324,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Shoes,Mixed Items,15,Australia,USER_S,5,Occasional,2013-11-01,2013-10-27,10000000163,2013-11-01,103324,10000010,27.48,1
-FP-GTC,15,14,108782,15,8,Auto Parts,2013-09-10 16:52:46,USER_Y,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,15,Australia,USER_S,14,Regular,2013-06-16,2013-06-16,10000000166,2013-06-16,108782,10000011,9.26,1
-Auction,0,12,80287,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Software,Office & Business,0,Ebay,USER_S,12,Merchant,2013-09-12,2013-09-08,10000000167,2013-09-12,80287,10000012,3.18,1
-Others,100,13,140746,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2013-09-28,2013-09-22,10000000086,2013-09-28,140746,10000013,3.18,1
-ABIN,0,14,87118,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Outdoor Sports,Paintball,0,Ebay,USER_S,14,Regular,2013-06-15,2013-06-09,10000000110,2013-06-15,87118,10000014,377.94,1
-Auction,0,12,25147,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,0,Ebay,USER_S,12,Merchant,2013-03-14,2013-03-10,10000000113,2013-03-14,25147,10000015,146.33,1
-FP-GTC,15,5,170302,15,15,Crafts,2010-09-13 07:13:34,USER_Y,Crafts,Embroidery,Design CDs,15,Australia,USER_S,5,Occasional,2013-09-01,2013-09-01,10000000069,2013-09-01,170302,10000016,51.23,1
-FP-non GTC,0,13,53064,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,0,Ebay,USER_S,13,Entrepreneur,2013-05-29,2013-05-26,10000000079,2013-05-29,53064,10000017,72.65,1
-Auction,0,13,132939,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Other,0,Ebay,USER_S,13,Entrepreneur,2013-05-31,2013-05-26,10000000080,2013-05-31,132939,10000018,66.6,1
-Auction,15,12,113593,15,40,Cell Phones & Portable Electro,2012-09-11 20:26:04,USER_Y,Phones,Mobile Phones,Mobile Phones,15,Australia,USER_S,12,Merchant,2013-03-18,2013-03-17,10000000130,2013-03-18,113593,10000019,9.26,1
-Auction,100,14,34273,100,8,Auto - Parts,2008-06-03 13:36:00,USER_Y,eBay Motors,Parts & Accessories,Motorcycle,100,Ebaymotors,USER_S,14,Regular,2013-07-19,2013-07-14,10000000268,2013-07-19,34273,10000020,583.44,1
-FP-GTC,15,14,106340,15,16,Home & Garden,2012-09-11 20:26:04,USER_Y,Home & Garden,Gardening,Hand Tools,15,Australia,USER_S,14,Regular,2013-06-23,2013-06-23,10000000132,2013-06-23,106340,10000021,638.72,1
-FP-GTC,15,14,150265,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Boys,15,Australia,USER_S,14,Regular,2013-05-20,2013-05-19,10000000115,2013-05-20,150265,10000022,4.54,1
-FP-GTC,0,12,24760,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,0,Ebay,USER_S,12,Merchant,2013-05-17,2013-05-12,10000000117,2013-05-17,24760,10000023,319.79,1
-Auction,0,12,37831,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Merchandise & Memorabilia,0,Ebay,USER_S,12,Merchant,2013-03-11,2013-03-10,10000000118,2013-03-11,37831,10000024,20.35,1
-FP-non GTC,3,5,1120,3,3,BookComics & Magazines,2008-06-03 13:36:00,USER_Y,Books,First Editions,Other,3,英国,USER_S,5,Occasional,2013-01-30,2013-01-27,10000000223,2013-01-30,1120,10000025,223.63,1
-FP-GTC,100,13,43972,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,ATV Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2013-01-26,2013-01-20,10000000224,2013-01-26,43972,10000026,204.28,1
-Auction,15,14,166013,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,15,Australia,USER_S,14,Regular,2013-03-22,2013-03-17,10000000243,2013-03-22,166013,10000027,5.48,1
-Auction,15,14,15568,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Unisex,15,Australia,USER_S,14,Regular,2013-07-23,2013-07-21,10000000217,2013-07-23,15568,10000028,27.48,1
-FP-GTC,15,5,103178,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Bags,Women's Bags,15,Australia,USER_S,5,Occasional,2013-07-27,2013-07-21,10000000218,2013-07-27,103178,10000029,21.72,1
-ABIN,0,12,2023,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Team Sports,Basketball,0,Ebay,USER_S,12,Merchant,2013-10-29,2013-10-27,10000000221,2013-10-29,2023,10000030,3.18,1
-FP-GTC,0,11,94847,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Car Video,0,Ebay,USER_S,11,Large Merchant,2013-10-08,2013-10-06,10000000256,2013-10-08,94847,10000031,491.32,1
-Auction,0,14,15868,0,23,Real Estate,2008-06-03 13:36:00,USER_Y,Real Estate,Land,Land,0,Ebay,USER_S,14,Regular,2013-04-26,2013-04-21,10000000257,2013-04-26,15868,10000032,448.8,1
-Auction,0,13,32876,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Improvement,Plumbing & Fixtures,0,Ebay,USER_S,13,Entrepreneur,2013-01-01,2013-01-01,10000000263,2013-01-01,32876,10000033,415.73,1
-Auction,0,13,62179,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2013-01-15,2013-01-13,10000000245,2013-01-15,62179,10000034,377.94,1
-FP-GTC,15,14,33038,15,30,Musical Instruments,2010-06-14 07:48:40,USER_Y,Musical Instruments,Instruments,Guitars (Electric),15,Australia,USER_S,14,Regular,2013-05-27,2013-05-26,10000000248,2013-05-27,33038,10000035,146.33,1
-FP-GTC,0,5,156614,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,5,Occasional,2013-11-11,2013-11-10,10000000254,2013-11-11,156614,10000036,7.12,1
-Auction,0,13,106246,0,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,0,Ebay,USER_S,13,Entrepreneur,2013-03-08,2013-03-03,10000000192,2013-03-08,106246,10000037,42.99,1
-Auction,0,13,20865,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2013-03-25,2013-03-24,10000000196,2013-03-25,20865,10000038,12.85,1
-FP-GTC,0,13,15115,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,13,Entrepreneur,2013-08-20,2013-08-18,10000000203,2013-08-20,15115,10000039,55.89,1
-FP-GTC,0,14,3838,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000179,2013-05-17,3838,10000040,73.26,1
-Auction,0,11,759,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,11,Large Merchant,2013-06-05,2013-06-02,10000000183,2013-06-05,759,10000041,112.56,1
-FP-non GTC,0,11,61323,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,0,Ebay,USER_S,11,Large Merchant,2013-10-08,2013-10-06,10000000185,2013-10-08,61323,10000042,3.49,1
-Auction,0,13,121153,0,2,Baby,2012-09-11 20:26:04,USER_Y,Baby,Nursery Decor,Night Lights,0,Ebay,USER_S,13,Entrepreneur,2013-08-14,2013-08-11,10000000211,2013-08-14,121153,10000043,184.21,1
-Auction,0,13,88750,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,0,Ebay,USER_S,13,Entrepreneur,2013-08-14,2013-08-11,10000000213,2013-08-14,88750,10000044,157.14,1
-FP-GTC,15,14,161567,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Laptop & Desktop Accessories,Laptop Batteries,15,Australia,USER_S,14,Regular,2013-05-17,2013-05-12,10000000214,2013-05-17,161567,10000045,72.65,1
-FP-GTC,15,14,113802,15,99,Everything Else,2008-06-03 13:36:00,USER_Y,Lots More...,Metaphysical,Herbs,15,Australia,USER_S,14,Regular,2013-08-09,2013-08-04,10000000204,2013-08-09,113802,10000046,51.23,1
-FP-non GTC,15,14,15808,15,9,Clothing & Accessories,2012-09-11 20:26:04,USER_Y,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,15,Australia,USER_S,14,Regular,2013-06-30,2013-06-30,10000000208,2013-06-30,15808,10000047,15.85,1
-FP-GTC,3,13,174053,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,External & Body Parts,3,英国,USER_S,13,Entrepreneur,2013-06-03,2013-06-02,10000000209,2013-06-03,174053,10000048,7.12,1
-Auction,0,14,2635,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Toy Soldiers,1970-Now,0,Ebay,USER_S,14,Regular,2013-12-31,2013-12-29,10000000042,2013-12-31,2635,10000049,12.04,1
-Auction,3,13,1161,3,18,DVDFilm & TV,2008-06-03 13:36:00,USER_Y,DVFilm & TV,Other Formats,Videos: NTSC (US),3,英国,USER_S,13,Entrepreneur,2013-12-25,2013-12-22,10000000044,2013-12-25,1161,10000050,73.26,1
-FP-non GTC,0,5,64076,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,0,Ebay,USER_S,5,Occasional,2013-03-28,2013-03-24,10000000022,2013-03-28,64076,10000051,184.21,1
-FP-GTC,15,13,33977,15,15,Crafts,2010-06-14 07:56:25,USER_Y,Crafts,Scrapbooking,Albums,15,Australia,USER_S,13,Entrepreneur,2013-01-30,2013-01-27,10000000023,2013-01-30,33977,10000052,172.03,1
-FP-GTC,0,14,31673,0,25,Sports Memorabilia,2010-02-08 10:48:39,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,0,Ebay,USER_S,14,Regular,2013-12-05,2013-12-01,10000000047,2013-12-05,31673,10000053,122.78,1
-Auction,3,14,174106,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,3,英国,USER_S,14,Regular,2013-10-08,2013-10-06,10000000056,2013-10-08,174106,10000054,92.98,1
-Auction,0,13,26249,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,0,Ebay,USER_S,13,Entrepreneur,2013-12-27,2013-12-22,10000000062,2013-12-27,26249,10000055,12.19,1
-FP-GTC,0,5,159184,0,24,Sporting Goods,2013-09-10 16:52:46,USER_Y,Sporting Goods,Winter Sports,Snowboarding,0,Ebay,USER_S,5,Occasional,2013-12-16,2013-12-15,10000000053,2013-12-16,159184,10000056,15.65,1
-FP-GTC,3,11,10058,3,27,Tickets,2013-09-10 16:52:46,USER_Y,Events Tickets,Other Tickets,Other Tickets,3,英国,USER_S,11,Large Merchant,2013-10-17,2013-10-13,10000000055,2013-10-17,10058,10000057,101.79,1
-ABIN,0,12,48904,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,0,Ebay,USER_S,12,Merchant,2013-11-17,2013-11-17,10000000020,2013-11-17,48904,10000058,7.12,1
-FP-non GTC,0,14,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,14,Regular,2013-09-18,2013-09-15,10000000007,2013-09-18,145970,10000059,12.85,1
-FP-GTC,0,13,963,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,0,Ebay,USER_S,13,Entrepreneur,2013-06-30,2013-06-30,10000000008,2013-06-30,963,10000060,12.19,1
-FP-GTC,3,13,118687,3,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Fragrances,Women's Fragrances,3,英国,USER_S,13,Entrepreneur,2013-10-12,2013-10-06,10000000002,2013-10-12,118687,10000061,92.98,1
-FP-GTC,0,14,20886,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,14,Regular,2013-08-20,2013-08-18,10000000003,2013-08-20,20886,10000062,42.99,1
-Auction,15,13,148324,15,31,Phones,2012-09-18 00:08:03,USER_XIANZHU,Phones,Mobile Accessories,CaseCoverSkins,15,Australia,USER_S,13,Entrepreneur,2013-08-29,2013-08-25,10000000010,2013-08-29,148324,10000063,1.88,1
-Auction,15,14,139255,15,17,JewelrGemWatches,2012-05-01 08:54:26,USER_Y,Jewellery & Watches,Fine Jewellery,Earrings,15,Australia,USER_S,14,Regular,2013-07-17,2013-07-14,10000000016,2013-07-17,139255,10000064,21.14,1
-FP-GTC,0,5,20213,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Postcards,US StateCities & Towns,0,Ebay,USER_S,5,Occasional,2013-07-23,2013-07-21,10000000017,2013-07-23,20213,10000065,21.14,1
-Auction,15,13,32996,15,42,Entertainment Memorabilia,2012-09-11 20:26:04,USER_Y,Movies,Television Memorabilia,Clippings,15,Australia,USER_S,13,Entrepreneur,2013-01-06,2013-01-06,10000000012,2013-01-06,32996,10000066,132.33,1
-FP-GTC,0,14,99985,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Trading Cards,Sci-FFantasy,0,Ebay,USER_S,14,Regular,2013-08-14,2013-08-11,10000000013,2013-08-14,99985,10000067,120.87,1
-Auction,3,14,67703,3,17,Jewellery & Watches,2012-06-19 21:15:09,USER_Y,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,3,英国,USER_S,14,Regular,2013-08-10,2013-08-04,10000000067,2013-08-10,67703,10000068,120.87,1
-FP-non GTC,0,11,65,0,11,Collectibles,2013-02-20 23:11:43,USER_Y,Collectibles,Comics,Platinum Age (1897-1937),0,Ebay,USER_S,11,Large Merchant,2013-09-28,2013-09-22,10000000085,2013-09-28,65,10000069,9.26,1
-FP-non GTC,0,14,130,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Transportation,Railroadiana & Trains,0,Ebay,USER_S,14,Regular,2013-08-21,2013-08-18,10000000141,2013-08-21,130,10000070,16.26,1
-FP-GTC,0,14,164,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,0,Ebay,USER_S,14,Regular,2013-03-11,2013-03-10,10000000078,2013-03-11,164,10000071,157.14,1
-FP-GTC,0,11,216,0,25,Sports Memorabilia,2013-02-20 23:11:43,USER_Y,Sports MeCards & Fan Shop,Cards,Hockey,0,Ebay,USER_S,11,Large Merchant,2013-05-06,2013-05-05,10000000109,2013-05-06,216,10000072,1.88,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-05-17,2013-05-12,10000000096,2013-05-17,223,10000073,12.04,1
-FP-non GTC,0,14,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,14,Regular,2013-01-10,2013-01-06,10000000095,2013-01-10,223,10000074,189.23,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-05-05,2013-05-05,10000000098,2013-05-05,223,10000075,73.26,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-02-03,2013-02-03,10000000097,2013-02-03,223,10000076,4.13,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-11-26,2013-11-24,10000000099,2013-11-26,223,10000077,290.72,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-08-30,2013-08-25,10000000100,2013-08-30,223,10000078,265.56,1
-FP-GTC,15,5,279,15,3,Books,2013-09-10 16:52:46,USER_Y,BookMagazines,Children's Books,Children's Books,15,Australia,USER_S,5,Occasional,2013-04-26,2013-04-21,10000000126,2013-04-26,279,10000079,5.91,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2013-06-30,2013-06-30,10000000252,2013-06-30,314,10000080,319.79,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2013-06-30,2013-06-30,10000000052,2013-06-30,314,10000081,246,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2013-12-16,2013-12-15,10000000253,2013-12-16,314,10000082,20.35,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2013-12-15,2013-12-15,10000000051,2013-12-15,314,10000083,36.7,1
-Auction,0,13,533,0,10,Coins,2012-06-19 21:15:09,USER_Y,Coins & Paper Money,Coins: World,Africa,0,Ebay,USER_S,13,Entrepreneur,2013-08-17,2013-08-11,10000000190,2013-08-17,533,10000084,101.79,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2013-12-15,2013-12-15,10000000251,2013-12-15,1349,10000085,47.71,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2013-02-04,2013-02-03,10000000050,2013-02-04,1349,10000086,3.49,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2013-01-11,2013-01-06,10000000049,2013-01-11,1349,10000087,46.44,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2013-02-04,2013-02-03,10000000250,2013-02-04,1349,10000088,4.54,1
-ABIN,0,14,1357,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000131,2013-05-17,1357,10000089,3.18,1
-FP-GTC,0,14,1504,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Electrical & Test Equipment,Test Equipment,0,Ebay,USER_S,14,Regular,2013-11-12,2013-11-10,10000000172,2013-11-12,1504,10000090,86.58,1
-FP-GTC,0,13,4943,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,13,Entrepreneur,2013-08-21,2013-08-18,10000000142,2013-08-21,4943,10000091,12.85,1
-ABIN,0,13,6762,0,-999,Unknown,2008-06-03 13:36:00,USER_Y,Unknown,Unknown,Unknown,0,Ebay,USER_S,13,Entrepreneur,2013-05-09,2013-05-05,10000000195,2013-05-09,6762,10000092,16.26,1
-Auction,3,13,9426,3,31,Mobile & Home Phones,2012-05-01 08:54:26,USER_Y,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,3,英国,USER_S,13,Entrepreneur,2013-09-19,2013-09-15,10000000070,2013-09-19,9426,10000093,21.14,1
-FP-non GTC,0,14,10866,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Animals,Farm & Countryside,0,Ebay,USER_S,14,Regular,2013-02-06,2013-02-03,10000000165,2013-02-06,10866,10000094,20.6,1
-Auction,0,13,11554,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Jeans,0,Ebay,USER_S,13,Entrepreneur,2013-02-02,2013-01-27,10000000187,2013-02-02,11554,10000095,246,1
-FP-GTC,0,14,11848,0,37,Health & Beauty,2012-06-19 21:15:09,USER_Y,Health & Beauty,Fragrances,Women,0,Ebay,USER_S,14,Regular,2013-08-23,2013-08-18,10000000189,2013-08-23,11848,10000096,109,1
-Auction,0,13,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,13,Entrepreneur,2013-08-03,2013-07-28,10000000139,2013-08-03,13836,10000097,39.41,1
-Auction,0,14,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000140,2013-05-17,13836,10000098,16.26,1
-FP-GTC,0,13,13987,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Paper,Booklets,0,Ebay,USER_S,13,Entrepreneur,2013-06-06,2013-06-02,10000000102,2013-06-06,13987,10000099,112.56,1
-Auction,0,14,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,14,Regular,2013-07-02,2013-06-30,10000000076,2013-07-02,15687,10000100,184.21,1
-Auction,0,11,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,11,Large Merchant,2013-10-25,2013-10-20,10000000082,2013-10-25,15687,10000001,27.48,1
-FP-non GTC,3,12,16145,3,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,3,英国,USER_S,12,Merchant,2013-04-20,2013-04-14,10000000129,2013-04-20,16145,10000002,26.45,1
-FP-non GTC,0,13,16145,0,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other,0,Ebay,USER_S,13,Entrepreneur,2013-03-12,2013-03-10,10000000128,2013-03-12,16145,10000003,415.73,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2013-03-28,2013-03-24,10000000222,2013-03-28,16509,10000004,56.36,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2013-10-29,2013-10-27,10000000021,2013-10-29,16509,10000005,2.44,1
-FP-GTC,0,14,20485,0,36,Home & Living,2012-05-01 08:54:26,USER_Y,Home & Garden,Furniture,Other,0,Ebay,USER_S,14,Regular,2013-05-22,2013-05-19,10000000134,2013-05-22,20485,10000006,269.76,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2013-01-25,2013-01-20,10000000135,2013-01-25,20485,10000007,109,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2013-06-12,2013-06-09,10000000136,2013-06-12,20485,10000008,101.79,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-26,2013-12-22,10000000241,2013-12-26,23446,10000009,246,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-26,2013-12-22,10000000041,2013-12-26,23446,10000010,189.23,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-31,2013-12-29,10000000242,2013-12-31,23446,10000011,15.65,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-10-04,2013-09-29,10000000040,2013-10-04,23446,10000012,28.23,1
-FP-GTC,0,5,24541,0,25,Sports Memorabilia,2013-07-16 08:23:09,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,0,Ebay,USER_S,5,Occasional,2013-03-16,2013-03-10,10000000194,2013-03-16,24541,10000013,16.26,1
-FP-GTC,0,5,26262,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Food & Beverage,0,Ebay,USER_S,5,Occasional,2013-05-21,2013-05-19,10000000101,2013-05-21,26262,10000014,122.78,1
-FP-GTC,3,14,30059,3,21,Photography,2012-09-11 20:26:04,USER_Y,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,3,英国,USER_S,14,Regular,2013-01-28,2013-01-27,10000000077,2013-01-28,30059,10000015,172.03,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,英国,USER_S,14,Regular,2013-04-26,2013-04-21,10000000057,2013-04-26,31387,10000016,42.99,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,英国,USER_S,14,Regular,2013-10-06,2013-10-06,10000000258,2013-10-06,31387,10000017,207.5,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000261,2013-11-06,31519,10000018,5.91,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,英国,USER_S,14,Regular,2013-10-06,2013-10-06,10000000058,2013-10-06,31519,10000019,39.41,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-12-28,2013-12-22,10000000059,2013-12-28,31519,10000020,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000060,2013-11-06,31519,10000021,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000260,2013-11-06,31519,10000022,78.48,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,英国,USER_S,14,Regular,2013-12-28,2013-12-22,10000000259,2013-12-28,31519,10000023,190.22,1
-FP-GTC,100,12,35570,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Motorcycle Parts,100,Ebaymotors,USER_S,12,Merchant,2013-06-11,2013-06-09,10000000156,2013-06-11,35570,10000024,2.44,1
-Auction,0,5,36250,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Hunting,Decoys,0,Ebay,USER_S,5,Occasional,2013-01-10,2013-01-06,10000000119,2013-01-10,36250,10000025,7.12,1
-FP-non GTC,0,14,38238,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Decor,Other,0,Ebay,USER_S,14,Regular,2013-09-17,2013-09-15,10000000186,2013-09-17,38238,10000026,36.7,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,英国,USER_S,14,Regular,2013-08-14,2013-08-11,10000000038,2013-08-14,40059,10000027,35.72,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,英国,USER_S,14,Regular,2013-08-09,2013-08-04,10000000239,2013-08-09,40059,10000028,3.49,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2013-12-02,2013-12-01,10000000034,2013-12-02,41940,10000029,223.63,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2013-02-01,2013-01-27,10000000235,2013-02-01,41940,10000030,265.56,1
-FP-non GTC,0,13,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,13,Entrepreneur,2013-07-28,2013-07-28,10000000127,2013-07-28,43479,10000031,62.02,1
-FP-GTC,0,12,44079,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,0,Ebay,USER_S,12,Merchant,2013-06-16,2013-06-16,10000000103,2013-06-16,44079,10000032,46.44,1
-Auction,101,14,45238,101,9,Vestiti ed Accessori,2012-09-11 20:27:22,USER_Y,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,101,Italy,USER_S,14,Regular,2013-08-23,2013-08-18,10000000201,2013-08-23,45238,10000033,132.33,1
-Auction,0,13,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,13,Entrepreneur,2013-06-15,2013-06-09,10000000122,2013-06-15,45333,10000034,448.8,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2013-06-15,2013-06-09,10000000123,2013-06-15,45333,10000035,207.5,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2013-06-01,2013-05-26,10000000124,2013-06-01,45333,10000036,190.22,1
-FP-GTC,0,14,46575,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Light Equipment & Tools,Air Tools,0,Ebay,USER_S,14,Regular,2013-08-10,2013-08-04,10000000072,2013-08-10,46575,10000037,16.71,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2013-03-22,2013-03-17,10000000043,2013-03-22,50508,10000038,4.13,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2013-12-25,2013-12-22,10000000244,2013-12-25,50508,10000039,1.88,1
-FP-GTC,0,13,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,13,Entrepreneur,2013-07-22,2013-07-21,10000000121,2013-07-22,50677,10000040,491.32,1
-FP-GTC,0,5,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,5,Occasional,2013-04-13,2013-04-07,10000000120,2013-04-13,50677,10000041,2.44,1
-Auction,0,14,51582,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),0,Ebay,USER_S,14,Regular,2013-04-16,2013-04-14,10000000168,2013-04-16,51582,10000042,56.36,1
-FP-GTC,0,13,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,13,Entrepreneur,2013-08-21,2013-08-18,10000000073,2013-08-21,57013,10000043,15.85,1
-FP-non GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2013-04-22,2013-04-21,10000000075,2013-04-22,57013,10000044,2.44,1
-FP-GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2013-08-29,2013-08-25,10000000074,2013-08-29,57013,10000045,7.12,1
-Auction,0,14,57784,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),0,Ebay,USER_S,14,Regular,2013-05-16,2013-05-12,10000000093,2013-05-16,57784,10000046,35.72,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,11,Large Merchant,2013-08-23,2013-08-18,10000000265,2013-08-23,57990,10000047,9.26,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2013-07-10,2013-07-07,10000000266,2013-07-10,57990,10000048,3.18,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2013-08-10,2013-08-04,10000000267,2013-08-10,57990,10000049,638.72,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2013-08-23,2013-08-18,10000000065,2013-08-23,57990,10000050,141.7,1
-ABIN,0,13,57990,0,9,Apparel,2013-09-10 16:52:46,USER_Y,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,0,Ebay,USER_S,13,Entrepreneur,2013-04-18,2013-04-14,10000000143,2013-04-18,57990,10000051,12.19,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2013-07-10,2013-07-07,10000000066,2013-07-10,57990,10000052,132.33,1
-ABIN,3,5,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,5,Occasional,2013-06-16,2013-06-16,10000000144,2013-06-16,57990,10000053,5.48,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,11,Large Merchant,2013-07-15,2013-07-14,10000000064,2013-07-15,57990,10000054,1.88,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000061,2013-11-06,60340,10000055,12.85,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2013-12-27,2013-12-22,10000000262,2013-12-27,60340,10000056,62.02,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2013-07-29,2013-07-28,10000000019,2013-07-29,60606,10000057,15.85,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2013-11-17,2013-11-17,10000000220,2013-11-17,60606,10000058,9.26,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2013-07-27,2013-07-21,10000000018,2013-07-27,60606,10000059,16.71,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2013-07-29,2013-07-28,10000000219,2013-07-29,60606,10000060,20.6,1
-Auction,3,5,63861,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Dresses,3,英国,USER_S,5,Occasional,2013-01-09,2013-01-06,10000000145,2013-01-09,63861,10000061,1.88,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2013-06-11,2013-06-09,10000000200,2013-06-11,63861,10000062,141.7,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2013-01-10,2013-01-06,10000000199,2013-01-10,63861,10000063,1.88,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2013-09-16,2013-09-15,10000000237,2013-09-16,63861,10000064,112.56,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2013-01-14,2013-01-13,10000000036,2013-01-14,63861,10000065,94.45,1
-Auction,0,14,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000125,2013-05-17,63861,10000066,78.48,1
-ABIN,0,13,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,13,Entrepreneur,2013-06-05,2013-06-02,10000000198,2013-06-05,63861,10000067,5.48,1
-Auction,3,14,63864,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Skirts,3,英国,USER_S,14,Regular,2013-05-24,2013-05-19,10000000094,2013-05-24,63864,10000068,28.23,1
-Others,0,13,63889,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,0,Ebay,USER_S,13,Entrepreneur,2013-05-15,2013-05-12,10000000104,2013-05-15,63889,10000069,3.49,1
-FP-GTC,2,11,67698,2,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewellery Packaging & Display,2,Canada,USER_S,11,Large Merchant,2013-03-25,2013-03-24,10000000107,2013-03-25,67698,10000070,15.65,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2013-03-09,2013-03-03,10000000108,2013-03-09,67698,10000071,5.48,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2013-12-05,2013-12-01,10000000106,2013-12-05,67698,10000072,246,1
-FP-non GTC,0,13,73506,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Tea PotSets,0,Ebay,USER_S,13,Entrepreneur,2013-04-18,2013-04-14,10000000182,2013-04-18,73506,10000073,122.78,1
-FP-GTC,0,14,75665,0,16,Home Improvement,2012-09-11 20:26:04,USER_Y,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,0,Ebay,USER_S,14,Regular,2013-11-01,2013-10-27,10000000169,2013-11-01,75665,10000074,223.63,1
-ABIN,3,5,75708,3,28,Toys & Games,2012-05-01 08:57:38,USER_Y,Toys & Games,Action Figures,TMovies & Video Games,3,英国,USER_S,5,Occasional,2013-05-03,2013-04-28,10000000146,2013-05-03,75708,10000075,141.7,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-04-21,2013-04-21,10000000151,2013-04-21,80053,10000076,21.14,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-03-12,2013-03-10,10000000149,2013-03-12,80053,10000077,55.89,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-05-19,2013-05-19,10000000150,2013-05-19,80053,10000078,51.23,1
-Auction,0,14,80135,0,12,Computers,2013-07-16 08:23:09,USER_Y,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,0,Ebay,USER_S,14,Regular,2013-11-23,2013-11-17,10000000083,2013-11-23,80135,10000079,21.72,1
-Auction,3,14,95672,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Shoes,Trainers,3,英国,USER_S,14,Regular,2013-10-19,2013-10-13,10000000089,2013-10-19,95672,10000080,204.28,1
-Others,0,11,95672,0,9,Apparel,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Women's Shoes,Athletic,0,Ebay,USER_S,11,Large Merchant,2013-05-18,2013-05-12,10000000152,2013-05-18,95672,10000081,21.14,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2013-02-01,2013-01-27,10000000035,2013-02-01,100847,10000082,204.28,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2013-01-14,2013-01-13,10000000236,2013-01-14,100847,10000083,122.78,1
-ABIN,3,14,139973,3,32,PC & Video Gaming,2012-09-11 20:26:04,USER_Y,Video Games & Consoles,Games,Games,3,英国,USER_S,14,Regular,2013-08-05,2013-08-04,10000000090,2013-08-05,139973,10000084,94.45,1
-ABIN,0,11,139973,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,11,Large Merchant,2013-05-19,2013-05-19,10000000091,2013-05-19,139973,10000085,86.58,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,英国,USER_S,14,Regular,2013-12-01,2013-12-01,10000000033,2013-12-01,150047,10000086,56.36,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,英国,USER_S,14,Regular,2013-12-02,2013-12-01,10000000234,2013-12-02,150047,10000087,290.72,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2013-01-11,2013-01-06,10000000249,2013-01-11,155226,10000088,60.37,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2013-05-27,2013-05-26,10000000048,2013-05-27,155226,10000089,112.56,1
-FP-GTC,0,13,156356,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Postcards,BuildingArchitecture,0,Ebay,USER_S,13,Entrepreneur,2013-09-01,2013-09-01,10000000181,2013-09-01,156356,10000090,265.56,1
-FP-GTC,0,11,158798,0,28,Toys,2008-09-09 22:08:47,USER_Y,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,0,Ebay,USER_S,11,Large Merchant,2013-04-11,2013-04-07,10000000092,2013-04-11,158798,10000091,35.72,1
-FP-non GTC,0,13,165888,0,17,JewelrGemWatches,2009-01-12 07:05:17,USER_Y,Jewelry & Watches,Vintage & Antique Jewelry,Costume,0,Ebay,USER_S,13,Entrepreneur,2013-05-05,2013-05-05,10000000191,2013-05-05,165888,10000092,92.98,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,英国,USER_S,11,Large Merchant,2013-11-21,2013-11-17,10000000229,2013-11-21,170083,10000093,28.23,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,英国,USER_S,11,Large Merchant,2013-10-07,2013-10-06,10000000028,2013-10-07,170083,10000094,27.48,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2013-07-12,2013-07-07,10000000031,2013-07-12,175750,10000095,9.26,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2013-06-07,2013-06-02,10000000032,2013-06-07,175750,10000096,3.18,1
-FP-GTC,0,14,175750,0,36,Home & Living,2012-05-01 08:57:38,USER_Y,Home & Garden,Bedding,Blankets & Throws,0,Ebay,USER_S,14,Regular,2013-05-22,2013-05-19,10000000177,2013-05-22,175750,10000097,12.04,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,13,Entrepreneur,2013-11-28,2013-11-24,10000000030,2013-11-28,175750,10000098,20.6,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,13,Entrepreneur,2013-07-12,2013-07-07,10000000231,2013-07-12,175750,10000099,12.04,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2013-06-07,2013-06-02,10000000232,2013-06-07,175750,10000100,4.13,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2013-12-01,2013-12-01,10000000233,2013-12-01,175750,10000201,73.26,1
-Auction,0,12,48027,0,10,Coins,2013-07-16 08:23:09,USER_Y,Coins & Paper Money,Paper Money: World,Asia,0,Ebay,USER_S,12,Merchant,2012-10-12,2012-10-06,10000000002,2012-10-12,48027,10000001,184.21,1
-Others,0,5,164261,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Earrings,0,Ebay,USER_S,5,Occasional,2012-08-20,2012-08-18,10000000003,2012-08-20,164261,10000002,172.03,1
-Auction,15,14,82494,15,3,Books,2010-09-13 07:05:54,USER_Y,BookMagazines,NULL,Comic Books,15,Australia,USER_S,14,Regular,2012-09-18,2012-09-15,10000000007,2012-09-18,82494,10000003,66.6,1
-Auction,15,12,66767,15,16,Home & Garden,2011-06-14 07:48:40,USER_Y,Home & Garden,NULL,Dogs,15,Australia,USER_S,12,Merchant,2012-06-30,2012-06-30,10000000008,2012-06-30,66767,10000004,92.98,1
-FP-GTC,0,5,152801,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,NULL,Earrings,0,Ebay,USER_S,5,Occasional,2012-08-29,2012-08-25,10000000010,2012-08-29,152801,10000005,132.33,1
-FP-GTC,0,13,43398,0,36,Home & Living,2013-02-20 23:11:43,USER_Y,Home & Garden,NULL,Cheese & Crackers,0,Ebay,USER_S,13,Entrepreneur,2012-01-06,2012-01-06,10000000012,2012-01-06,43398,10000006,7.12,1
-Auction,0,14,95173,0,37,Health & Beauty,2008-06-03 13:36:00,USER_Y,Health & Beauty,Bath & Body,Bath Sets & Kits,0,Ebay,USER_S,14,Regular,2012-08-14,2012-08-11,10000000013,2012-08-14,95173,10000007,204.28,1
-Auction,15,13,158666,15,28,Toys,2012-09-11 20:27:22,USER_Y,ToyHobbies,Action Figures,Anime & Manga,15,Australia,USER_S,13,Entrepreneur,2012-07-17,2012-07-14,10000000016,2012-07-17,158666,10000008,35.72,1
-Auction,0,12,12688,0,57,Sothebys,2008-06-03 13:36:00,USER_Y,eBay Premier,Books & Manuscripts,Books: Other,0,Ebay,USER_S,12,Merchant,2012-07-23,2012-07-21,10000000017,2012-07-23,12688,10000009,4.13,1
-FP-GTC,15,5,103324,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Shoes,Mixed Items,15,Australia,USER_S,5,Occasional,2012-07-27,2012-07-21,10000000018,2012-07-27,103324,10000010,27.48,1
-FP-GTC,15,14,108782,15,8,Auto Parts,2013-09-10 16:52:46,USER_Y,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,15,Australia,USER_S,14,Regular,2012-07-29,2012-07-28,10000000019,2012-07-29,108782,10000011,9.26,1
-Auction,0,12,80287,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Software,Office & Business,0,Ebay,USER_S,12,Merchant,2012-11-17,2012-11-17,10000000020,2012-11-17,80287,10000012,3.18,1
-Others,100,13,140746,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2012-10-29,2012-10-27,10000000021,2012-10-29,140746,10000013,3.18,1
-ABIN,0,14,87118,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Outdoor Sports,Paintball,0,Ebay,USER_S,14,Regular,2012-03-28,2012-03-24,10000000022,2012-03-28,87118,10000014,377.94,1
-Auction,0,12,25147,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,0,Ebay,USER_S,12,Merchant,2012-01-30,2012-01-27,10000000023,2012-01-30,25147,10000015,146.33,1
-FP-GTC,15,5,170302,15,15,Crafts,2010-09-13 07:13:34,USER_Y,Crafts,Embroidery,Design CDs,15,Australia,USER_S,5,Occasional,2012-10-07,2012-10-06,10000000028,2012-10-07,170302,10000016,51.23,1
-FP-non GTC,0,13,53064,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,0,Ebay,USER_S,13,Entrepreneur,2012-11-28,2012-11-24,10000000030,2012-11-28,53064,10000017,72.65,1
-Auction,0,13,132939,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Other,0,Ebay,USER_S,13,Entrepreneur,2012-07-12,2012-07-07,10000000031,2012-07-12,132939,10000018,66.6,1
-Auction,15,12,113593,15,40,Cell Phones & Portable Electro,2012-09-11 20:26:04,USER_Y,Phones,Mobile Phones,Mobile Phones,15,Australia,USER_S,12,Merchant,2012-06-07,2012-06-02,10000000032,2012-06-07,113593,10000019,9.26,1
-Auction,100,14,34273,100,8,Auto - Parts,2008-06-03 13:36:00,USER_Y,eBay Motors,Parts & Accessories,Motorcycle,100,Ebaymotors,USER_S,14,Regular,2012-12-01,2012-12-01,10000000033,2012-12-01,34273,10000020,583.44,1
-FP-GTC,15,14,106340,15,16,Home & Garden,2012-09-11 20:26:04,USER_Y,Home & Garden,Gardening,Hand Tools,15,Australia,USER_S,14,Regular,2012-12-02,2012-12-01,10000000034,2012-12-02,106340,10000021,638.72,1
-FP-GTC,15,14,150265,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Boys,15,Australia,USER_S,14,Regular,2012-02-01,2012-01-27,10000000035,2012-02-01,150265,10000022,4.54,1
-FP-GTC,0,12,24760,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,0,Ebay,USER_S,12,Merchant,2012-01-14,2012-01-13,10000000036,2012-01-14,24760,10000023,319.79,1
-Auction,0,12,37831,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Merchandise & Memorabilia,0,Ebay,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000038,2012-08-14,37831,10000024,20.35,1
-FP-non GTC,3,5,1120,3,3,BookComics & Magazines,2008-06-03 13:36:00,USER_Y,Books,First Editions,Other,3,英国,USER_S,5,Occasional,2012-10-04,2012-09-29,10000000040,2012-10-04,1120,10000025,223.63,1
-FP-GTC,100,13,43972,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,ATV Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2012-12-26,2012-12-22,10000000041,2012-12-26,43972,10000026,204.28,1
-Auction,15,14,166013,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,15,Australia,USER_S,14,Regular,2012-12-31,2012-12-29,10000000042,2012-12-31,166013,10000027,5.48,1
-Auction,15,14,15568,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Unisex,15,Australia,USER_S,14,Regular,2012-03-22,2012-03-17,10000000043,2012-03-22,15568,10000028,27.48,1
-FP-GTC,15,5,103178,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Bags,Women's Bags,15,Australia,USER_S,5,Occasional,2012-12-25,2012-12-22,10000000044,2012-12-25,103178,10000029,21.72,1
-ABIN,0,12,2023,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Team Sports,Basketball,0,Ebay,USER_S,12,Merchant,2012-12-05,2012-12-01,10000000047,2012-12-05,2023,10000030,3.18,1
-FP-GTC,0,11,94847,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Car Video,0,Ebay,USER_S,11,Large Merchant,2012-05-27,2012-05-26,10000000048,2012-05-27,94847,10000031,491.32,1
-Auction,0,14,15868,0,23,Real Estate,2008-06-03 13:36:00,USER_Y,Real Estate,Land,Land,0,Ebay,USER_S,14,Regular,2012-01-11,2012-01-06,10000000049,2012-01-11,15868,10000032,448.8,1
-Auction,0,13,32876,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Improvement,Plumbing & Fixtures,0,Ebay,USER_S,13,Entrepreneur,2012-02-04,2012-02-03,10000000050,2012-02-04,32876,10000033,415.73,1
-Auction,0,13,62179,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2012-12-15,2012-12-15,10000000051,2012-12-15,62179,10000034,377.94,1
-FP-GTC,15,14,33038,15,30,Musical Instruments,2010-06-14 07:48:40,USER_Y,Musical Instruments,Instruments,Guitars (Electric),15,Australia,USER_S,14,Regular,2012-06-30,2012-06-30,10000000052,2012-06-30,33038,10000035,146.33,1
-FP-GTC,0,5,156614,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,5,Occasional,2012-12-16,2012-12-15,10000000053,2012-12-16,156614,10000036,7.12,1
-Auction,0,13,106246,0,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,0,Ebay,USER_S,13,Entrepreneur,2012-10-17,2012-10-13,10000000055,2012-10-17,106246,10000037,42.99,1
-Auction,0,13,20865,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000056,2012-10-08,20865,10000038,12.85,1
-FP-GTC,0,13,15115,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,13,Entrepreneur,2012-04-26,2012-04-21,10000000057,2012-04-26,15115,10000039,55.89,1
-FP-GTC,0,14,3838,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,0,Ebay,USER_S,14,Regular,2012-10-06,2012-10-06,10000000058,2012-10-06,3838,10000040,73.26,1
-Auction,0,11,759,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,11,Large Merchant,2012-12-28,2012-12-22,10000000059,2012-12-28,759,10000041,112.56,1
-FP-non GTC,0,11,61323,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,0,Ebay,USER_S,11,Large Merchant,2012-11-06,2012-11-03,10000000060,2012-11-06,61323,10000042,3.49,1
-Auction,0,13,121153,0,2,Baby,2012-09-11 20:26:04,USER_Y,Baby,Nursery Decor,Night Lights,0,Ebay,USER_S,13,Entrepreneur,2012-11-06,2012-11-03,10000000061,2012-11-06,121153,10000043,184.21,1
-Auction,0,13,88750,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,0,Ebay,USER_S,13,Entrepreneur,2012-12-27,2012-12-22,10000000062,2012-12-27,88750,10000044,157.14,1
-FP-GTC,15,14,161567,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Laptop & Desktop Accessories,Laptop Batteries,15,Australia,USER_S,14,Regular,2012-07-15,2012-07-14,10000000064,2012-07-15,161567,10000045,72.65,1
-FP-GTC,15,14,113802,15,99,Everything Else,2008-06-03 13:36:00,USER_Y,Lots More...,Metaphysical,Herbs,15,Australia,USER_S,14,Regular,2012-08-23,2012-08-18,10000000065,2012-08-23,113802,10000046,51.23,1
-FP-non GTC,15,14,15808,15,9,Clothing & Accessories,2012-09-11 20:26:04,USER_Y,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,15,Australia,USER_S,14,Regular,2012-07-10,2012-07-07,10000000066,2012-07-10,15808,10000047,15.85,1
-FP-GTC,3,13,174053,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,External & Body Parts,3,英国,USER_S,13,Entrepreneur,2012-08-10,2012-08-04,10000000067,2012-08-10,174053,10000048,7.12,1
-Auction,0,14,2635,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Toy Soldiers,1970-Now,0,Ebay,USER_S,14,Regular,2012-09-01,2012-09-01,10000000069,2012-09-01,2635,10000049,12.04,1
-Auction,3,13,1161,3,18,DVDFilm & TV,2008-06-03 13:36:00,USER_Y,DVFilm & TV,Other Formats,Videos: NTSC (US),3,英国,USER_S,13,Entrepreneur,2012-09-19,2012-09-15,10000000070,2012-09-19,1161,10000050,73.26,1
-FP-non GTC,0,5,64076,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,0,Ebay,USER_S,5,Occasional,2012-08-10,2012-08-04,10000000072,2012-08-10,64076,10000051,184.21,1
-FP-GTC,15,13,33977,15,15,Crafts,2010-06-14 07:56:25,USER_Y,Crafts,Scrapbooking,Albums,15,Australia,USER_S,13,Entrepreneur,2012-08-21,2012-08-18,10000000073,2012-08-21,33977,10000052,172.03,1
-FP-GTC,0,14,31673,0,25,Sports Memorabilia,2010-02-08 10:48:39,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,0,Ebay,USER_S,14,Regular,2012-08-29,2012-08-25,10000000074,2012-08-29,31673,10000053,122.78,1
-Auction,3,14,174106,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,3,英国,USER_S,14,Regular,2012-04-22,2012-04-21,10000000075,2012-04-22,174106,10000054,92.98,1
-Auction,0,13,26249,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,0,Ebay,USER_S,13,Entrepreneur,2012-07-02,2012-06-30,10000000076,2012-07-02,26249,10000055,12.19,1
-FP-GTC,0,5,159184,0,24,Sporting Goods,2013-09-10 16:52:46,USER_Y,Sporting Goods,Winter Sports,Snowboarding,0,Ebay,USER_S,5,Occasional,2012-01-28,2012-01-27,10000000077,2012-01-28,159184,10000056,15.65,1
-FP-GTC,3,11,10058,3,27,Tickets,2013-09-10 16:52:46,USER_Y,Events Tickets,Other Tickets,Other Tickets,3,英国,USER_S,11,Large Merchant,2012-03-11,2012-03-10,10000000078,2012-03-11,10058,10000057,101.79,1
-ABIN,0,12,48904,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,0,Ebay,USER_S,12,Merchant,2012-05-29,2012-05-26,10000000079,2012-05-29,48904,10000058,7.12,1
-FP-non GTC,0,14,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,14,Regular,2012-05-31,2012-05-26,10000000080,2012-05-31,145970,10000059,12.85,1
-FP-GTC,0,13,963,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,0,Ebay,USER_S,13,Entrepreneur,2012-10-25,2012-10-20,10000000082,2012-10-25,963,10000060,12.19,1
-FP-GTC,3,13,118687,3,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Fragrances,Women's Fragrances,3,英国,USER_S,13,Entrepreneur,2012-11-23,2012-11-17,10000000083,2012-11-23,118687,10000061,92.98,1
-FP-GTC,0,14,20886,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,14,Regular,2012-09-28,2012-09-22,10000000085,2012-09-28,20886,10000062,42.99,1
-Auction,15,13,148324,15,31,Phones,2012-09-18 00:08:03,USER_XIANZHU,Phones,Mobile Accessories,CaseCoverSkins,15,Australia,USER_S,13,Entrepreneur,2012-09-28,2012-09-22,10000000086,2012-09-28,148324,10000063,1.88,1
-Auction,15,14,139255,15,17,JewelrGemWatches,2012-05-01 08:54:26,USER_Y,Jewellery & Watches,Fine Jewellery,Earrings,15,Australia,USER_S,14,Regular,2012-10-19,2012-10-13,10000000089,2012-10-19,139255,10000064,21.14,1
-FP-GTC,0,5,20213,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Postcards,US StateCities & Towns,0,Ebay,USER_S,5,Occasional,2012-08-05,2012-08-04,10000000090,2012-08-05,20213,10000065,21.14,1
-Auction,15,13,32996,15,42,Entertainment Memorabilia,2012-09-11 20:26:04,USER_Y,Movies,Television Memorabilia,Clippings,15,Australia,USER_S,13,Entrepreneur,2012-05-19,2012-05-19,10000000091,2012-05-19,32996,10000066,132.33,1
-FP-GTC,0,14,99985,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Trading Cards,Sci-FFantasy,0,Ebay,USER_S,14,Regular,2012-04-11,2012-04-07,10000000092,2012-04-11,99985,10000067,120.87,1
-Auction,3,14,67703,3,17,Jewellery & Watches,2012-06-19 21:15:09,USER_Y,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,3,英国,USER_S,14,Regular,2012-05-16,2012-05-12,10000000093,2012-05-16,67703,10000068,120.87,1
-FP-non GTC,0,11,65,0,11,Collectibles,2013-02-20 23:11:43,USER_Y,Collectibles,Comics,Platinum Age (1897-1937),0,Ebay,USER_S,11,Large Merchant,2012-05-24,2012-05-19,10000000094,2012-05-24,65,10000069,9.26,1
-FP-non GTC,0,14,130,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Transportation,Railroadiana & Trains,0,Ebay,USER_S,14,Regular,2012-01-10,2012-01-06,10000000095,2012-01-10,130,10000070,16.26,1
-FP-GTC,0,14,164,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000096,2012-05-17,164,10000071,157.14,1
-FP-GTC,0,11,216,0,25,Sports Memorabilia,2013-02-20 23:11:43,USER_Y,Sports MeCards & Fan Shop,Cards,Hockey,0,Ebay,USER_S,11,Large Merchant,2012-02-03,2012-02-03,10000000097,2012-02-03,216,10000072,1.88,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-05-05,2012-05-05,10000000098,2012-05-05,223,10000073,12.04,1
-FP-non GTC,0,14,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,14,Regular,2012-11-26,2012-11-24,10000000099,2012-11-26,223,10000074,189.23,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-08-30,2012-08-25,10000000100,2012-08-30,223,10000075,73.26,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-05-21,2012-05-19,10000000101,2012-05-21,223,10000076,4.13,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-06-06,2012-06-02,10000000102,2012-06-06,223,10000077,290.72,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-06-16,2012-06-16,10000000103,2012-06-16,223,10000078,265.56,1
-FP-GTC,15,5,279,15,3,Books,2013-09-10 16:52:46,USER_Y,BookMagazines,Children's Books,Children's Books,15,Australia,USER_S,5,Occasional,2012-05-15,2012-05-12,10000000104,2012-05-15,279,10000079,5.91,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2012-12-05,2012-12-01,10000000106,2012-12-05,314,10000080,319.79,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2012-03-25,2012-03-24,10000000107,2012-03-25,314,10000081,246,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2012-03-09,2012-03-03,10000000108,2012-03-09,314,10000082,20.35,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2012-05-06,2012-05-05,10000000109,2012-05-06,314,10000083,36.7,1
-Auction,0,13,533,0,10,Coins,2012-06-19 21:15:09,USER_Y,Coins & Paper Money,Coins: World,Africa,0,Ebay,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000110,2012-06-15,533,10000084,101.79,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2012-03-14,2012-03-10,10000000113,2012-03-14,1349,10000085,47.71,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2012-05-20,2012-05-19,10000000115,2012-05-20,1349,10000086,3.49,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2012-05-17,2012-05-12,10000000117,2012-05-17,1349,10000087,46.44,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2012-03-11,2012-03-10,10000000118,2012-03-11,1349,10000088,4.54,1
-ABIN,0,14,1357,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,14,Regular,2012-01-10,2012-01-06,10000000119,2012-01-10,1357,10000089,3.18,1
-FP-GTC,0,14,1504,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Electrical & Test Equipment,Test Equipment,0,Ebay,USER_S,14,Regular,2012-04-13,2012-04-07,10000000120,2012-04-13,1504,10000090,86.58,1
-FP-GTC,0,13,4943,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,13,Entrepreneur,2012-07-22,2012-07-21,10000000121,2012-07-22,4943,10000091,12.85,1
-ABIN,0,13,6762,0,-999,Unknown,2008-06-03 13:36:00,USER_Y,Unknown,Unknown,Unknown,0,Ebay,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000122,2012-06-15,6762,10000092,16.26,1
-Auction,3,13,9426,3,31,Mobile & Home Phones,2012-05-01 08:54:26,USER_Y,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,3,英国,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000123,2012-06-15,9426,10000093,21.14,1
-FP-non GTC,0,14,10866,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Animals,Farm & Countryside,0,Ebay,USER_S,14,Regular,2012-06-01,2012-05-26,10000000124,2012-06-01,10866,10000094,20.6,1
-Auction,0,13,11554,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Jeans,0,Ebay,USER_S,13,Entrepreneur,2012-05-17,2012-05-12,10000000125,2012-05-17,11554,10000095,246,1
-FP-GTC,0,14,11848,0,37,Health & Beauty,2012-06-19 21:15:09,USER_Y,Health & Beauty,Fragrances,Women,0,Ebay,USER_S,14,Regular,2012-04-26,2012-04-21,10000000126,2012-04-26,11848,10000096,109,1
-Auction,0,13,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,13,Entrepreneur,2012-07-28,2012-07-28,10000000127,2012-07-28,13836,10000097,39.41,1
-Auction,0,14,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,14,Regular,2012-03-12,2012-03-10,10000000128,2012-03-12,13836,10000098,16.26,1
-FP-GTC,0,13,13987,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Paper,Booklets,0,Ebay,USER_S,13,Entrepreneur,2012-04-20,2012-04-14,10000000129,2012-04-20,13987,10000099,112.56,1
-Auction,0,14,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,14,Regular,2012-03-18,2012-03-17,10000000130,2012-03-18,15687,10000100,184.21,1
-Auction,0,11,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,11,Large Merchant,2012-05-17,2012-05-12,10000000131,2012-05-17,15687,10000001,27.48,1
-FP-non GTC,3,12,16145,3,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,3,英国,USER_S,12,Merchant,2012-06-23,2012-06-23,10000000132,2012-06-23,16145,10000002,26.45,1
-FP-non GTC,0,13,16145,0,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other,0,Ebay,USER_S,13,Entrepreneur,2012-05-22,2012-05-19,10000000134,2012-05-22,16145,10000003,415.73,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2012-01-25,2012-01-20,10000000135,2012-01-25,16509,10000004,56.36,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2012-06-12,2012-06-09,10000000136,2012-06-12,16509,10000005,2.44,1
-FP-GTC,0,14,20485,0,36,Home & Living,2012-05-01 08:54:26,USER_Y,Home & Garden,Furniture,Other,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000137,2012-05-17,20485,10000006,269.76,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2012-08-03,2012-07-28,10000000139,2012-08-03,20485,10000007,109,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2012-05-17,2012-05-12,10000000140,2012-05-17,20485,10000008,101.79,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-08-21,2012-08-18,10000000141,2012-08-21,23446,10000009,246,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-08-21,2012-08-18,10000000142,2012-08-21,23446,10000010,189.23,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-04-18,2012-04-14,10000000143,2012-04-18,23446,10000011,15.65,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-06-16,2012-06-16,10000000144,2012-06-16,23446,10000012,28.23,1
-FP-GTC,0,5,24541,0,25,Sports Memorabilia,2013-07-16 08:23:09,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,0,Ebay,USER_S,5,Occasional,2012-01-09,2012-01-06,10000000145,2012-01-09,24541,10000013,16.26,1
-FP-GTC,0,5,26262,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Food & Beverage,0,Ebay,USER_S,5,Occasional,2012-05-03,2012-04-28,10000000146,2012-05-03,26262,10000014,122.78,1
-FP-GTC,3,14,30059,3,21,Photography,2012-09-11 20:26:04,USER_Y,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,3,英国,USER_S,14,Regular,2012-05-20,2012-05-19,10000000147,2012-05-20,30059,10000015,172.03,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,英国,USER_S,14,Regular,2012-03-12,2012-03-10,10000000149,2012-03-12,31387,10000016,42.99,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,英国,USER_S,14,Regular,2012-05-19,2012-05-19,10000000150,2012-05-19,31387,10000017,207.5,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-04-21,2012-04-21,10000000151,2012-04-21,31519,10000018,5.91,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,英国,USER_S,14,Regular,2012-05-18,2012-05-12,10000000152,2012-05-18,31519,10000019,39.41,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-06-16,2012-06-16,10000000155,2012-06-16,31519,10000020,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-06-11,2012-06-09,10000000156,2012-06-11,31519,10000021,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-03-31,2012-03-31,10000000157,2012-03-31,31519,10000022,78.48,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,英国,USER_S,14,Regular,2012-11-12,2012-11-10,10000000158,2012-11-12,31519,10000023,190.22,1
-FP-GTC,100,12,35570,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Motorcycle Parts,100,Ebaymotors,USER_S,12,Merchant,2012-04-06,2012-03-31,10000000161,2012-04-06,35570,10000024,2.44,1
-Auction,0,5,36250,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Hunting,Decoys,0,Ebay,USER_S,5,Occasional,2012-11-01,2012-10-27,10000000163,2012-11-01,36250,10000025,7.12,1
-FP-non GTC,0,14,38238,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Decor,Other,0,Ebay,USER_S,14,Regular,2012-02-06,2012-02-03,10000000165,2012-02-06,38238,10000026,36.7,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,英国,USER_S,14,Regular,2012-06-16,2012-06-16,10000000166,2012-06-16,40059,10000027,35.72,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,英国,USER_S,14,Regular,2012-09-12,2012-09-08,10000000167,2012-09-12,40059,10000028,3.49,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2012-04-16,2012-04-14,10000000168,2012-04-16,41940,10000029,223.63,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2012-11-01,2012-10-27,10000000169,2012-11-01,41940,10000030,265.56,1
-FP-non GTC,0,13,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,13,Entrepreneur,2012-06-14,2012-06-09,10000000170,2012-06-14,43479,10000031,62.02,1
-FP-GTC,0,12,44079,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,0,Ebay,USER_S,12,Merchant,2012-11-12,2012-11-10,10000000172,2012-11-12,44079,10000032,46.44,1
-Auction,101,14,45238,101,9,Vestiti ed Accessori,2012-09-11 20:27:22,USER_Y,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,101,Italy,USER_S,14,Regular,2012-03-22,2012-03-17,10000000173,2012-03-22,45238,10000033,132.33,1
-Auction,0,13,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,13,Entrepreneur,2012-05-22,2012-05-19,10000000177,2012-05-22,45333,10000034,448.8,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2012-03-10,2012-03-10,10000000178,2012-03-10,45333,10000035,207.5,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000179,2012-05-17,45333,10000036,190.22,1
-FP-GTC,0,14,46575,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Light Equipment & Tools,Air Tools,0,Ebay,USER_S,14,Regular,2012-09-01,2012-09-01,10000000181,2012-09-01,46575,10000037,16.71,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2012-04-18,2012-04-14,10000000182,2012-04-18,50508,10000038,4.13,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2012-06-05,2012-06-02,10000000183,2012-06-05,50508,10000039,1.88,1
-FP-GTC,0,13,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000185,2012-10-08,50677,10000040,491.32,1
-FP-GTC,0,5,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,5,Occasional,2012-09-17,2012-09-15,10000000186,2012-09-17,50677,10000041,2.44,1
-Auction,0,14,51582,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),0,Ebay,USER_S,14,Regular,2012-02-02,2012-01-27,10000000187,2012-02-02,51582,10000042,56.36,1
-FP-GTC,0,13,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,13,Entrepreneur,2012-08-23,2012-08-18,10000000189,2012-08-23,57013,10000043,15.85,1
-FP-non GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2012-08-17,2012-08-11,10000000190,2012-08-17,57013,10000044,2.44,1
-FP-GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2012-05-05,2012-05-05,10000000191,2012-05-05,57013,10000045,7.12,1
-Auction,0,14,57784,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),0,Ebay,USER_S,14,Regular,2012-03-08,2012-03-03,10000000192,2012-03-08,57784,10000046,35.72,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,11,Large Merchant,2012-03-16,2012-03-10,10000000194,2012-03-16,57990,10000047,9.26,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2012-05-09,2012-05-05,10000000195,2012-05-09,57990,10000048,3.18,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2012-03-25,2012-03-24,10000000196,2012-03-25,57990,10000049,638.72,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2012-06-05,2012-06-02,10000000198,2012-06-05,57990,10000050,141.7,1
-ABIN,0,13,57990,0,9,Apparel,2013-09-10 16:52:46,USER_Y,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,0,Ebay,USER_S,13,Entrepreneur,2012-01-10,2012-01-06,10000000199,2012-01-10,57990,10000051,12.19,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,14,Regular,2012-06-11,2012-06-09,10000000200,2012-06-11,57990,10000052,132.33,1
-ABIN,3,5,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,5,Occasional,2012-08-23,2012-08-18,10000000201,2012-08-23,57990,10000053,5.48,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,英国,USER_S,11,Large Merchant,2012-08-20,2012-08-18,10000000203,2012-08-20,57990,10000054,1.88,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2012-08-09,2012-08-04,10000000204,2012-08-09,60340,10000055,12.85,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2012-06-30,2012-06-30,10000000208,2012-06-30,60340,10000056,62.02,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2012-06-03,2012-06-02,10000000209,2012-06-03,60606,10000057,15.85,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000211,2012-08-14,60606,10000058,9.26,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000213,2012-08-14,60606,10000059,16.71,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,英国,USER_S,12,Merchant,2012-05-17,2012-05-12,10000000214,2012-05-17,60606,10000060,20.6,1
-Auction,3,5,63861,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Dresses,3,英国,USER_S,5,Occasional,2012-07-23,2012-07-21,10000000217,2012-07-23,63861,10000061,1.88,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2012-07-27,2012-07-21,10000000218,2012-07-27,63861,10000062,141.7,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2012-07-29,2012-07-28,10000000219,2012-07-29,63861,10000063,1.88,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2012-11-17,2012-11-17,10000000220,2012-11-17,63861,10000064,112.56,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2012-10-29,2012-10-27,10000000221,2012-10-29,63861,10000065,94.45,1
-Auction,0,14,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,14,Regular,2012-03-28,2012-03-24,10000000222,2012-03-28,63861,10000066,78.48,1
-ABIN,0,13,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,13,Entrepreneur,2012-01-30,2012-01-27,10000000223,2012-01-30,63861,10000067,5.48,1
-Auction,3,14,63864,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Skirts,3,英国,USER_S,14,Regular,2012-01-26,2012-01-20,10000000224,2012-01-26,63864,10000068,28.23,1
-Others,0,13,63889,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,0,Ebay,USER_S,13,Entrepreneur,2012-11-21,2012-11-17,10000000229,2012-11-21,63889,10000069,3.49,1
-FP-GTC,2,11,67698,2,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewellery Packaging & Display,2,Canada,USER_S,11,Large Merchant,2012-07-12,2012-07-07,10000000231,2012-07-12,67698,10000070,15.65,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2012-06-07,2012-06-02,10000000232,2012-06-07,67698,10000071,5.48,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2012-12-01,2012-12-01,10000000233,2012-12-01,67698,10000072,246,1
-FP-non GTC,0,13,73506,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Tea PotSets,0,Ebay,USER_S,13,Entrepreneur,2012-12-02,2012-12-01,10000000234,2012-12-02,73506,10000073,122.78,1
-FP-GTC,0,14,75665,0,16,Home Improvement,2012-09-11 20:26:04,USER_Y,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,0,Ebay,USER_S,14,Regular,2012-02-01,2012-01-27,10000000235,2012-02-01,75665,10000074,223.63,1
-ABIN,3,5,75708,3,28,Toys & Games,2012-05-01 08:57:38,USER_Y,Toys & Games,Action Figures,TMovies & Video Games,3,英国,USER_S,5,Occasional,2012-01-14,2012-01-13,10000000236,2012-01-14,75708,10000075,141.7,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-09-16,2012-09-15,10000000237,2012-09-16,80053,10000076,21.14,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-08-09,2012-08-04,10000000239,2012-08-09,80053,10000077,55.89,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-12-26,2012-12-22,10000000241,2012-12-26,80053,10000078,51.23,1
-Auction,0,14,80135,0,12,Computers,2013-07-16 08:23:09,USER_Y,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,0,Ebay,USER_S,14,Regular,2012-12-31,2012-12-29,10000000242,2012-12-31,80135,10000079,21.72,1
-Auction,3,14,95672,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Shoes,Trainers,3,英国,USER_S,14,Regular,2012-03-22,2012-03-17,10000000243,2012-03-22,95672,10000080,204.28,1
-Others,0,11,95672,0,9,Apparel,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Women's Shoes,Athletic,0,Ebay,USER_S,11,Large Merchant,2012-12-25,2012-12-22,10000000244,2012-12-25,95672,10000081,21.14,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2012-01-15,2012-01-13,10000000245,2012-01-15,100847,10000082,204.28,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2012-05-27,2012-05-26,10000000248,2012-05-27,100847,10000083,122.78,1
-ABIN,3,14,139973,3,32,PC & Video Gaming,2012-09-11 20:26:04,USER_Y,Video Games & Consoles,Games,Games,3,英国,USER_S,14,Regular,2012-01-11,2012-01-06,10000000249,2012-01-11,139973,10000084,94.45,1
-ABIN,0,11,139973,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,11,Large Merchant,2012-02-04,2012-02-03,10000000250,2012-02-04,139973,10000085,86.58,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,英国,USER_S,14,Regular,2012-12-15,2012-12-15,10000000251,2012-12-15,150047,10000086,56.36,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,英国,USER_S,14,Regular,2012-06-30,2012-06-30,10000000252,2012-06-30,150047,10000087,290.72,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2012-12-16,2012-12-15,10000000253,2012-12-16,155226,10000088,60.37,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2012-11-11,2012-11-10,10000000254,2012-11-11,155226,10000089,112.56,1
-FP-GTC,0,13,156356,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Postcards,BuildingArchitecture,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000256,2012-10-08,156356,10000090,265.56,1
-FP-GTC,0,11,158798,0,28,Toys,2008-09-09 22:08:47,USER_Y,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,0,Ebay,USER_S,11,Large Merchant,2012-04-26,2012-04-21,10000000257,2012-04-26,158798,10000091,35.72,1
-FP-non GTC,0,13,165888,0,17,JewelrGemWatches,2009-01-12 07:05:17,USER_Y,Jewelry & Watches,Vintage & Antique Jewelry,Costume,0,Ebay,USER_S,13,Entrepreneur,2012-10-06,2012-10-06,10000000258,2012-10-06,165888,10000092,92.98,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,英国,USER_S,11,Large Merchant,2012-12-28,2012-12-22,10000000259,2012-12-28,170083,10000093,28.23,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,英国,USER_S,11,Large Merchant,2012-11-06,2012-11-03,10000000260,2012-11-06,170083,10000094,27.48,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2012-11-06,2012-11-03,10000000261,2012-11-06,175750,10000095,9.26,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2012-12-27,2012-12-22,10000000262,2012-12-27,175750,10000096,3.18,1
-FP-GTC,0,14,175750,0,36,Home & Living,2012-05-01 08:57:38,USER_Y,Home & Garden,Bedding,Blankets & Throws,0,Ebay,USER_S,14,Regular,2012-01-01,2012-01-01,10000000263,2012-01-01,175750,10000097,12.04,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,13,Entrepreneur,2012-08-23,2012-08-18,10000000265,2012-08-23,175750,10000098,20.6,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,13,Entrepreneur,2012-07-10,2012-07-07,10000000266,2012-07-10,175750,10000099,12.04,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2012-08-10,2012-08-04,10000000267,2012-08-10,175750,10000100,4.13,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,英国,USER_S,14,Regular,2012-07-19,2012-07-14,10000000268,2012-07-19,175750,10000201,73.26,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9355165b-06ad-4c04-977c-a251e66e7e98.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9355165b-06ad-4c04-977c-a251e66e7e98.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9355165b-06ad-4c04-977c-a251e66e7e98.dict
deleted file mode 100644
index a1c248c..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9355165b-06ad-4c04-977c-a251e66e7e98.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict
deleted file mode 100644
index aa42f96..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict
deleted file mode 100644
index bfcd8e7..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict and /dev/null differ
[39/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
deleted file mode 100644
index 17d62e7..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.model.*;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * @author George Song (ysong1)
- */
-public class Cuboid implements Comparable<Cuboid> {
-
- private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
-
- public static Cuboid findById(CubeDesc cube, byte[] cuboidID) {
- return findById(cube, Bytes.toLong(cuboidID));
- }
-
- public static Cuboid findById(CubeDesc cube, long cuboidID) {
- Map<Long, Cuboid> cubeCache = CUBOID_CACHE.get(cube.getName());
- if (cubeCache == null) {
- cubeCache = new ConcurrentHashMap<Long, Cuboid>();
- CUBOID_CACHE.put(cube.getName(), cubeCache);
- }
- Cuboid cuboid = cubeCache.get(cuboidID);
- if (cuboid == null) {
- long validCuboidID = translateToValidCuboid(cube, cuboidID);
- if (Cuboid.isValid(cube, validCuboidID) == false) {
- throw new RuntimeException("Didn't find a valid cuboid: " + validCuboidID);
- }
-
- cuboid = new Cuboid(cube, cuboidID, validCuboidID);
- cubeCache.put(cuboidID, cuboid);
- }
- return cuboid;
-
- }
-
- public static boolean isValid(CubeDesc cube, long cuboidID) {
- RowKeyDesc rowkey = cube.getRowkey();
-
- if (cuboidID < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
- }
-
- if (checkBaseCuboid(rowkey, cuboidID)) {
- return true;
- }
-
- if (checkMandatoryColumns(rowkey, cuboidID) == false) {
- return false;
- }
-
- if (checkAggregationGroup(rowkey, cuboidID) == false) {
- return false;
- }
-
- if (checkHierarchy(rowkey, cuboidID) == false) {
- return false;
- }
-
- return true;
- }
-
- public static long getBaseCuboidId(CubeDesc cube) {
- return cube.getRowkey().getFullMask();
- }
-
- public static Cuboid getBaseCuboid(CubeDesc cube) {
- return findById(cube, getBaseCuboidId(cube));
- }
-
- private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) {
- // add mandantory
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
- if (cuboidID < mandatoryColumnMask) {
- cuboidID = cuboidID | mandatoryColumnMask;
- }
-
- // add hierarchy
- for (DimensionDesc dimension : cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- boolean found = false;
- long result = 0;
- if (hierarchies != null && hierarchies.length > 0) {
- for (int i = hierarchies.length - 1; i >= 0; i--) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = rowkey.getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((rowkey.getTailMask() & bit) > 0)
- continue; // ignore levels in tail, they don't participate
-
- if ((bit & cuboidID) > 0) {
- found = true;
- }
-
- if (found == true) {
- result = result | bit;
- }
- }
- cuboidID = cuboidID | result;
- }
- }
-
- // find the left-most aggregation group
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
-
- if (leftover == 0) {
- return cuboidID;
- }
-
- if (leftover != 0) {
- cuboidID = cuboidID | mask.leftoverMask;
- return cuboidID;
- }
- }
- }
-
- // doesn't have column in aggregation groups
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- if (leftover == 0) {
- // doesn't have column in tail group
- if (cuboidWithoutMandatory != 0) {
- return cuboidID;
- } else {
- // no column except mandatory, add one column
- cuboidID = cuboidID | Long.lowestOneBit(rowkey.getAggrGroupFullMask());
- return translateToValidCuboid(cubeDesc, cuboidID);
- }
- }
-
- // has column in tail group
- cuboidID = cuboidID | rowkey.getTailMask();
- return cuboidID;
-
- }
-
- private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
- long baseCuboidId = rowkey.getFullMask();
- if (cuboidID > baseCuboidId) {
- throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
- }
- return baseCuboidId == cuboidID;
- }
-
- private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
- long mandatoryColumnMask = rowkey.getMandatoryColumnMask();
-
- // note the all-zero cuboid (except for mandatory) is not valid
- if (cuboidID <= mandatoryColumnMask)
- return false;
-
- return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
- }
-
- private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
- List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
- // if no hierarchy defined in metadata
- if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
- return true;
- }
-
- hier: for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
- long result = cuboidID & hierarchyMasks.fullMask;
- if (result > 0) {
- // if match one of the hierarchy constrains, return true;
- for (long mask : hierarchyMasks.allMasks) {
- if (result == mask) {
- continue hier;
- }
- }
- return false;
- }
- }
- return true;
- }
-
- private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
- long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
- long leftover;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
- leftover = cuboidWithoutMandatory & ~mask.groupMask;
- return leftover == 0 || leftover == mask.leftoverMask;
- }
- }
-
- leftover = cuboidWithoutMandatory & rowkey.getTailMask();
- return leftover == 0 || leftover == rowkey.getTailMask();
- }
-
- private CubeDesc cube;
- private final long inputID;
- private final long id;
- private final byte[] idBytes;
- private final boolean requirePostAggregation;
- private List<TblColRef> dimensionColumns;
-
- // will translate the cuboidID if it is not valid
- private Cuboid(CubeDesc cube, long originalID, long validID) {
- this.cube = cube;
- this.inputID = originalID;
- this.id = validID;
- this.idBytes = Bytes.toBytes(id);
- this.dimensionColumns = translateIdToColumns(this.id);
- this.requirePostAggregation = calcExtraAggregation(this.inputID, this.id) != 0;
- }
-
- private List<TblColRef> translateIdToColumns(long cuboidID) {
- List<TblColRef> dimesnions = new ArrayList<TblColRef>();
- RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
- for (int i = 0; i < allColumns.length; i++) {
- // NOTE: the order of column in list!!!
- long bitmask = 1L << allColumns[i].getBitIndex();
- if ((cuboidID & bitmask) != 0) {
- TblColRef colRef = allColumns[i].getColRef();
- dimesnions.add(colRef);
- }
- }
- return dimesnions;
- }
-
- private long calcExtraAggregation(long inputID, long id) {
- long diff = id ^ inputID;
- return eliminateHierarchyAggregation(diff);
- }
-
- // higher level in hierarchy can be ignored when counting aggregation columns
- private long eliminateHierarchyAggregation(long id) {
- List<HierarchyMask> hierarchyMaskList = cube.getRowkey().getHierarchyMasks();
- if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) {
- for (HierarchyMask hierMask : hierarchyMaskList) {
- long[] allMasks = hierMask.allMasks;
- for (int i = allMasks.length - 1; i > 0; i--) {
- long bit = allMasks[i] ^ allMasks[i - 1];
- if ((inputID & bit) != 0) {
- id &= ~allMasks[i - 1];
- }
- }
- }
- }
- return id;
- }
-
- public CubeDesc getCube() {
- return cube;
- }
-
- public List<TblColRef> getColumns() {
- return dimensionColumns;
- }
-
- public List<TblColRef> getAggregationColumns() {
- long aggrColsID = eliminateHierarchyAggregation(id);
- return translateIdToColumns(aggrColsID);
- }
-
- public long getId() {
- return id;
- }
-
- public byte[] getBytes() {
- return idBytes;
- }
-
- public long getInputID() {
- return inputID;
- }
-
- public boolean useAncestor() {
- return inputID != id;
- }
-
- public boolean requirePostAggregation() {
- return requirePostAggregation;
- }
-
- public static void clearCache() {
- CUBOID_CACHE.clear();
- }
-
- public static void reloadCache(String cubeDescName) {
- CUBOID_CACHE.remove(cubeDescName);
- }
-
- @Override
- public String toString() {
- return "Cuboid [id=" + id + "]";
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (id ^ (id >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Cuboid other = (Cuboid) obj;
- if (id != other.id)
- return false;
- return true;
- }
-
- @Override
- public int compareTo(Cuboid o) {
- if (this.id < o.id) {
- return -1;
- } else if (this.id > o.id) {
- return 1;
- } else {
- return 0;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
deleted file mode 100644
index 7bb40f6..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-import org.apache.kylin.cube.model.RowKeyDesc.HierarchyMask;
-
-/**
- * @author yangli9
- *
- */
-public class CuboidCLI {
-
- public static void main(String[] args) throws IOException {
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- if ("test".equals(args[0])) {
- CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]);
- simulateCuboidGeneration(cubeDesc);
- }
- }
-
- public static int simulateCuboidGeneration(CubeDesc cube) {
- CuboidScheduler scheduler = new CuboidScheduler(cube);
-
- long baseCuboid = Cuboid.getBaseCuboidId(cube);
- Collection<Long> cuboidSet = new TreeSet<Long>();
- cuboidSet.add(baseCuboid);
- LinkedList<Long> cuboidQueue = new LinkedList<Long>();
- cuboidQueue.push(baseCuboid);
- while (!cuboidQueue.isEmpty()) {
- long cuboid = cuboidQueue.pop();
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
- for (Long sc : spnanningCuboids) {
- boolean notfound = cuboidSet.add(sc);
- if (!notfound) {
- throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
- }
- cuboidQueue.push(sc);
- }
- }
-
- /** disable this due to poor performance when dimension number is big
- TreeSet<Long> enumCuboids = enumCalcCuboidCount(cube);
- if (enumCuboids.equals(cuboidSet) == false) {
- throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
- }
- **/
-
- int mathCount = mathCalcCuboidCount(cube);
- if (mathCount != cuboidSet.size()) {
- throw new IllegalStateException("Math cuboid count " + mathCount + ", but actual cuboid count " + cuboidSet.size() + ", make sure aggregation groups has no duplication.");
- }
-
- return mathCount;
-
- }
-
- public static TreeSet<Long> enumCalcCuboidCount(CubeDesc cube) {
- long baseCuboid = Cuboid.getBaseCuboidId(cube);
- TreeSet<Long> expectedCuboids = new TreeSet<Long>();
- for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) {
- if (Cuboid.isValid(cube, cuboid)) {
- expectedCuboids.add(cuboid);
- }
- }
- return expectedCuboids;
- }
-
- public static int[] calculateAllLevelCount(CubeDesc cube) {
- int levels = cube.getRowkey().getNCuboidBuildLevels();
- int[] allLevelCounts = new int[levels + 1];
-
- CuboidScheduler scheduler = new CuboidScheduler(cube);
- LinkedList<Long> nextQueue = new LinkedList<Long>();
- LinkedList<Long> currentQueue = new LinkedList<Long>();
- long baseCuboid = Cuboid.getBaseCuboidId(cube);
- currentQueue.push(baseCuboid);
-
- for (int i = 0; i <= levels; i++) {
- allLevelCounts[i] = currentQueue.size();
- while (!currentQueue.isEmpty()) {
- long cuboid = currentQueue.pop();
- Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
- nextQueue.addAll(spnanningCuboids);
- }
- currentQueue = nextQueue;
- nextQueue = new LinkedList<Long>();
- }
-
- return allLevelCounts;
- }
-
- public static int mathCalcCuboidCount(CubeDesc cube) {
- int result = 1; // 1 for base cuboid
-
- RowKeyDesc rowkey = cube.getRowkey();
- AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
- result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
- }
-
- return result;
- }
-
- private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
- long groupMask = aggrGroupMask.groupMask;
- int n = mathCalcCuboidCount_combination(rowkey, groupMask);
- n -= 2; // exclude group all 1 and all 0
-
- long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
- if (nonUniqueMask > 0) {
- // exclude duplicates caused by non-unique columns
- // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
- n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
- }
-
- if (hasTail) {
- n *= 2; // tail being 1 and 0
- n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
- }
-
- return n;
- }
-
- private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
- if (colMask == 0) // no column selected
- return 0;
-
- int count = 1;
-
- for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
- long hierBits = colMask & hierMask.fullMask;
- if (hierBits != 0) {
- count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
- colMask &= ~hierBits;
- }
- }
-
- count *= Math.pow(2, Long.bitCount(colMask));
-
- return count;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
deleted file mode 100644
index 026d898..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cuboid;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.RowKeyDesc.AggrGroupMask;
-
-public class CuboidScheduler {
-
- private final CubeDesc cubeDef;
- private final int size;
- private final long max;
- private final Map<Long, Collection<Long>> cache;
-
- public CuboidScheduler(CubeDesc cube) {
- this.cubeDef = cube;
- this.size = cube.getRowkey().getRowKeyColumns().length;
- this.max = (long) Math.pow(2, size) - 1;
- this.cache = new ConcurrentHashMap<Long, Collection<Long>>();
- }
-
- public Collection<Long> getSpanningCuboid(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cuboid " + cuboid + " is out of scope 0-" + max);
- }
-
- Collection<Long> result = cache.get(cuboid);
- if (result != null) {
- return result;
- }
-
- // smaller sibling's children
- Collection<Long> allPrevOffspring = new HashSet<Long>();
- for (Long sibling : findSmallerSibling(cuboid)) {
- Collection<Long> prevOffsprings = generateChildren(sibling);
- allPrevOffspring.addAll(prevOffsprings);
- }
-
- // my children is my generation excluding smaller sibling's generation
- result = new ArrayList<Long>();
- for (Long offspring : generateChildren(cuboid)) {
- if (!allPrevOffspring.contains(offspring)) {
- result.add(offspring);
- }
- }
-
- cache.put(cuboid, result);
- return result;
- }
-
- private Collection<Long> generateChildren(long cuboid) {
- Collection<Long> result = new HashSet<Long>();
-
- // generate zero tail cuboid -- the one with all 1 in the first
- // aggregation group and all 0 for the rest bits
- generateZeroTailBase(cuboid, result);
-
- RowKeyDesc rowkey = cubeDef.getRowkey();
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if (belongTo(cuboidWithoutMandatory, mask) == false)
- continue;
-
- long[] groupOneBitMasks = mask.groupOneBitMasks;
- for (int i = 0; i < groupOneBitMasks.length; i++) {
- long oneBit = groupOneBitMasks[i];
- if ((cuboid & oneBit) == 0)
- continue;
-
- long child = cuboid ^ oneBit;
- if (Cuboid.isValid(cubeDef, child)) {
- result.add(child);
- }
- }
-
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
- }
-
- return result;
- }
-
- private void generateZeroTailBase(long cuboid, Collection<Long> result) {
- RowKeyDesc rowkey = cubeDef.getRowkey();
-
- long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
-
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
- long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
- if (zeroTail > 0 && zeroTail != cuboid) {
- result.add(zeroTail);
- }
- }
- if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
- break;
- }
- }
-
- public Collection<Long> findSmallerSibling(long cuboid) {
- if (!Cuboid.isValid(cubeDef, cuboid)) {
- return Collections.emptyList();
- }
-
- RowKeyDesc rowkey = cubeDef.getRowkey();
-
- // do combination in all related groups
- long groupAllBitMask = 0;
- for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
- if ((mask.groupMask & cuboid) > 0) {
- groupAllBitMask |= mask.groupMask;
- }
- }
-
- long groupBitValue = cuboid & groupAllBitMask;
- long leftBitValue = cuboid & ~groupAllBitMask;
- long[] groupOneBits = bits(groupAllBitMask);
-
- Collection<Long> siblings = new HashSet<Long>();
- combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
- return siblings;
- }
-
- private long[] bits(long groupAllBitMask) {
- int size = Long.bitCount(groupAllBitMask);
- long[] r = new long[size];
- long l = groupAllBitMask;
- int i = 0;
- while (l != 0) {
- long bit = Long.highestOneBit(l);
- r[i++] = bit;
- l ^= bit;
- }
- return r;
- }
-
- private void combination(long cuboid, Collection<Long> siblings, long[] bitMasks, int offset, long bitValue, int k) {
- if (k == 0) {
- if (Cuboid.isValid(cubeDef, bitValue)) {
- siblings.add(bitValue);
- }
- } else {
- for (int i = offset; i < bitMasks.length; i++) {
- long newBitValue = bitValue | bitMasks[i];
- if (newBitValue < cuboid) {
- combination(cuboid, siblings, bitMasks, i + 1, newBitValue, k - 1);
- }
- }
- }
- }
-
- private boolean belongTo(long cuboidWithoutMandatory, AggrGroupMask mask) {
- long groupBits = cuboidWithoutMandatory & mask.groupMask;
- long leftoverBits = cuboidWithoutMandatory & mask.leftoverMask;
- return groupBits > 0 && (leftoverBits == 0 || leftoverBits == mask.leftoverMask);
- }
-
- public int getCardinality(long cuboid) {
- if (cuboid > max || cuboid < 0) {
- throw new IllegalArgumentException("Cubiod " + cuboid + " is out of scope 0-" + max);
- }
-
- return Long.bitCount(cuboid);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/estimation/CubeSizeEstimationCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/estimation/CubeSizeEstimationCLI.java b/cube/src/main/java/org/apache/kylin/cube/estimation/CubeSizeEstimationCLI.java
deleted file mode 100644
index 188b157..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/estimation/CubeSizeEstimationCLI.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.estimation;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.cuboid.CuboidScheduler;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- * Created by honma on 9/1/14.
- */
-public class CubeSizeEstimationCLI {
-
- public static class RowKeyColInfo {
- public List<List<Integer>> hierachyColBitIndice;
- public List<Integer> nonHierachyColBitIndice;
- }
-
- public static long estimatedCubeSize(String cubeName, long[] cardinality) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeManager = CubeManager.getInstance(config);
- CubeInstance cubeInstance = cubeManager.getCube(cubeName);
- CubeDesc cubeDesc = cubeInstance.getDescriptor();
-
- CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
- long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
- LinkedList<Long> cuboidQueue = new LinkedList<Long>();
- cuboidQueue.push(baseCuboid);
-
- long totalSpace = 0;
-
- while (!cuboidQueue.isEmpty()) {
- long cuboidID = cuboidQueue.pop();
- Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
- for (Long sc : spanningCuboid) {
- cuboidQueue.push(sc);
- }
-
- totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
- }
- return totalSpace;
- }
-
- public static long estimateCuboidSpace(long cuboidID, long[] cardinality, CubeDesc cubeDesc) {
-
- RowKeyColInfo rowKeyColInfo = extractRowKeyInfo(cubeDesc);
- RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
-
- long rowCount = 1;
- int[] rowKeySpaces = estimateRowKeyColSpace(rowKeyDesc, cardinality);
- int dimensionSpace = 0;
- int measureSpace = getMeasureSpace(cubeDesc);
-
- for (List<Integer> hlist : rowKeyColInfo.hierachyColBitIndice) {
- // for hierachy columns, the cardinality of the most detailed column
- // nominates.
- int i;
- for (i = 0; i < hlist.size() && rowKeyColExists(hlist.get(i), cuboidID); ++i) {
- dimensionSpace += rowKeySpaces[hlist.get(i)];
- }
-
- if (i != 0)
- rowCount *= cardinality[hlist.get(i - 1)];
- }
-
- for (int index : rowKeyColInfo.nonHierachyColBitIndice) {
- if (rowKeyColExists(index, cuboidID)) {
- rowCount *= cardinality[index];
- dimensionSpace += rowKeySpaces[index];
- }
- }
- return rowCount * (dimensionSpace + measureSpace);
- }
-
- private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) {
- RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns();
- int[] ret = new int[rowKeyColDescs.length];
- for (int i = 0; i < rowKeyColDescs.length; ++i) {
- RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i];
- if (rowKeyColDesc.getDictionary() == null) {
- if (rowKeyColDesc.getLength() == 0)
- throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0");
- ret[i] = rowKeyColDesc.getLength();
- } else {
- ret[i] = estimateDictionaryColSpace(cardinality[i]);
- }
- }
- return ret;
- }
-
- // TODO what if it's date dictionary?
- private static int estimateDictionaryColSpace(long cardinality) {
- long mask = 1L;
- int i;
- for (i = Long.SIZE - 1; i >= 0; i--) {
- if ((cardinality & (mask << i)) != 0) {
- break;
- }
- }
-
- if (i < 0)
- throw new IllegalStateException("the cardinality is 0");
-
- return ((i + 1) + 7) / 8;// the bytes required to save at most
- // cardinality numbers
- }
-
- private static int getMeasureSpace(CubeDesc cubeDesc) {
- int space = 0;
- for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
- DataType returnType = measureDesc.getFunction().getReturnDataType();
- space += returnType.getStorageBytesEstimate();
- }
- return space;
- }
-
- private static boolean rowKeyColExists(int bitIndex, long cuboidID) {
- long mask = 1L << bitIndex;
- return (cuboidID & mask) != 0;
- }
-
- private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
- RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
- RowKeyColInfo info = new RowKeyColInfo();
- info.hierachyColBitIndice = new ArrayList<List<Integer>>();
- info.nonHierachyColBitIndice = new ArrayList<Integer>();
- HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();
-
- for (DimensionDesc dim : cubeDesc.getDimensions()) {
- if (dim.getHierarchy() != null) {
- LinkedList<Integer> hlist = new LinkedList<Integer>();
- for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
- int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
- hlist.add(index);
- heirachyIndexSet.add(index);
- }
- info.hierachyColBitIndice.add(hlist);
- }
- }
-
- for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
- if (!heirachyIndexSet.contains(i)) {
- info.nonHierachyColBitIndice.add(i);
- }
- }
-
- return info;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
deleted file mode 100644
index 532950b..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.Map;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * @author xjiang
- *
- */
-public abstract class AbstractRowKeyEncoder {
-
- public static final byte DEFAULT_BLANK_BYTE = Dictionary.NULL;
-
- protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class);
-
- public static AbstractRowKeyEncoder createInstance(CubeSegment cubeSeg, Cuboid cuboid) {
- return new RowKeyEncoder(cubeSeg, cuboid);
- }
-
- protected final Cuboid cuboid;
- protected byte blankByte = DEFAULT_BLANK_BYTE;
-
- protected AbstractRowKeyEncoder(Cuboid cuboid) {
- this.cuboid = cuboid;
- }
-
- public void setBlankByte(byte blankByte) {
- this.blankByte = blankByte;
- }
-
- abstract public byte[] encode(Map<TblColRef, String> valueMap);
-
- abstract public byte[] encode(byte[][] values);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyKeyEncoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyKeyEncoder.java
deleted file mode 100644
index f44affd..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyKeyEncoder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.Arrays;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-
-/**
- *
- * @author xjiang
- *
- */
-public class FuzzyKeyEncoder extends RowKeyEncoder {
-
- public FuzzyKeyEncoder(CubeSegment seg, Cuboid cuboid) {
- super(seg, cuboid);
- }
-
- @Override
- protected byte[] defaultValue(int length) {
- byte[] keyBytes = new byte[length];
- Arrays.fill(keyBytes, RowConstants.FUZZY_MASK_ZERO);
- return keyBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
deleted file mode 100644
index 6aae631..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.Arrays;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class FuzzyMaskEncoder extends RowKeyEncoder {
-
- public FuzzyMaskEncoder(CubeSegment seg, Cuboid cuboid) {
- super(seg, cuboid);
- }
-
- @Override
- protected int fillHeader(byte[] bytes, byte[][] values) {
- // always fuzzy match cuboid ID to lock on the selected cuboid
- int cuboidStart = this.headerLength - RowConstants.ROWKEY_CUBOIDID_LEN;
- Arrays.fill(bytes, 0, cuboidStart, RowConstants.FUZZY_MASK_ONE);
- Arrays.fill(bytes, cuboidStart, this.headerLength, RowConstants.FUZZY_MASK_ZERO);
- return this.headerLength;
- }
-
- @Override
- protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
- if (value == null) {
- Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, RowConstants.FUZZY_MASK_ONE);
- } else {
- Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, RowConstants.FUZZY_MASK_ZERO);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
deleted file mode 100644
index 6f5a20f..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowConstants {
-
- // row key fixed length place holder
- public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
- // row key lower bound
- public static final byte ROWKEY_LOWER_BYTE = 0;
- // row key upper bound
- public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff;
- // row key cuboid id length
- public static final int ROWKEY_CUBOIDID_LEN = 8;
-
- // fuzzy mask
- public static final byte FUZZY_MASK_ZERO = 0;
- public static final byte FUZZY_MASK_ONE = 1;
-
- // row value delimiter
- public static final byte ROWVALUE_DELIMITER_BYTE = 7;
- public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7);
- public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 };
-
- public static final int ROWVALUE_BUFFER_SIZE = 1024 * 1024; // 1 MB
-
- // marker class
- public static final byte[][] BYTE_ARR_MARKER = new byte[0][];
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
deleted file mode 100644
index 00ecd46..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Read/Write column values from/into bytes
- *
- * @author yangli9
- */
-@SuppressWarnings("unchecked")
-public class RowKeyColumnIO {
-
- private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
-
- private ISegment ISegment;
-
- public RowKeyColumnIO(ISegment ISegment) {
- this.ISegment = ISegment;
- }
-
- public int getColumnLength(TblColRef col) {
- return ISegment.getColumnLength(col);
- }
-
- //TODO is type cast really necessary here?
- public Dictionary<String> getDictionary(TblColRef col) {
- return (Dictionary<String>) ISegment.getDictionary(col);
- }
-
- public void writeColumn(TblColRef column, byte[] value, int valueLen, byte dft, byte[] output, int outputOffset) {
- writeColumn(column, value, valueLen, 0, dft, output, outputOffset);
- }
-
- public void writeColumn(TblColRef column, byte[] value, int valueLen, int roundingFlag, byte dft, byte[] output, int outputOffset) {
-
- Dictionary<String> dict = getDictionary(column);
- int columnLen = getColumnLength(column);
-
- // non-dict value
- if (dict == null) {
- byte[] valueBytes = padFixLen(columnLen, value);
- System.arraycopy(valueBytes, 0, output, outputOffset, columnLen);
- return;
- }
-
- // dict value
- try {
- int id = dict.getIdFromValueBytes(value, 0, valueLen, roundingFlag);
- BytesUtil.writeUnsigned(id, output, outputOffset, dict.getSizeOfId());
- } catch (IllegalArgumentException ex) {
- for (int i = outputOffset; i < outputOffset + columnLen; i++)
- output[i] = dft;
- logger.error("Can't translate value " + Bytes.toString(value, 0, valueLen) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", dft));
- }
- }
-
- private byte[] padFixLen(int length, byte[] valueBytes) {
- int valLen = valueBytes.length;
- if (valLen == length) {
- return valueBytes;
- } else if (valLen < length) {
- byte[] newValueBytes = new byte[length];
- System.arraycopy(valueBytes, 0, newValueBytes, 0, valLen);
- Arrays.fill(newValueBytes, valLen, length, RowConstants.ROWKEY_PLACE_HOLDER_BYTE);
- return newValueBytes;
- } else {
- return Arrays.copyOf(valueBytes, length);
- }
- }
-
- public String readColumnString(TblColRef col, byte[] bytes, int bytesLen) {
- Dictionary<String> dict = getDictionary(col);
- if (dict == null) {
- bytes = Bytes.head(bytes, bytesLen);
- if (isNull(bytes)) {
- return null;
- }
- bytes = removeFixLenPad(bytes, 0);
- return Bytes.toString(bytes);
- } else {
- int id = BytesUtil.readUnsigned(bytes, 0, bytesLen);
- try {
- String value = dict.getValueFromId(id);
- return value;
- } catch (IllegalArgumentException e) {
- logger.error("Can't get dictionary value for column " + col.getName() + " (id = " + id + ")");
- return "";
- }
- }
- }
-
- private boolean isNull(byte[] bytes) {
- // all 0xFF is NULL
- if (bytes.length == 0)
- return false;
- for (int i = 0; i < bytes.length; i++) {
- if (bytes[i] != AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE)
- return false;
- }
- return true;
- }
-
- private byte[] removeFixLenPad(byte[] bytes, int offset) {
- int padCount = 0;
- for (int i = offset; i < bytes.length; i++) {
- byte vb = bytes[i];
- if (vb == RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
- padCount++;
- }
- }
-
- int size = bytes.length - offset - padCount;
- byte[] stripBytes = new byte[size];
- int index = 0;
- for (int i = offset; i < bytes.length; i++) {
- byte vb = bytes[i];
- if (vb != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
- stripBytes[index++] = vb;
- }
- }
- return stripBytes;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
deleted file mode 100644
index 64bc813..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.Collection;
-import java.util.Comparator;
-
-import org.apache.kylin.metadata.datatype.DataType;
-
-/**
- * @author yangli9
- */
-abstract public class RowKeyColumnOrder implements Comparator<String> {
-
- public static final NumberOrder NUMBER_ORDER = new NumberOrder();
- public static final StringOrder STRING_ORDER = new StringOrder();
-
- public static RowKeyColumnOrder getInstance(DataType type) {
- if (type.isNumberFamily())
- return NUMBER_ORDER;
- else
- return STRING_ORDER;
- }
-
- public String max(Collection<String> values) {
- String max = null;
- for (String v : values) {
- if (max == null || compare(max, v) < 0)
- max = v;
- }
- return max;
- }
-
- public String min(Collection<String> values) {
- String min = null;
- for (String v : values) {
- if (min == null || compare(min, v) > 0)
- min = v;
- }
- return min;
- }
-
- public String min(String v1, String v2) {
- if (v1 == null)
- return v2;
- else if (v2 == null)
- return v1;
- else
- return compare(v1, v2) <= 0 ? v1 : v2;
- }
-
- public String max(String v1, String v2) {
- if (v1 == null)
- return v2;
- else if (v2 == null)
- return v1;
- else
- return compare(v1, v2) >= 0 ? v1 : v2;
- }
-
- @Override
- public int compare(String o1, String o2) {
- // consider null
- if (o1 == o2)
- return 0;
- if (o1 == null)
- return -1;
- if (o2 == null)
- return 1;
-
- return compareNonNull(o1, o2);
- }
-
- abstract int compareNonNull(String o1, String o2);
-
- private static class StringOrder extends RowKeyColumnOrder {
- @Override
- public int compareNonNull(String o1, String o2) {
- return o1.compareTo(o2);
- }
- }
-
- private static class NumberOrder extends RowKeyColumnOrder {
- @Override
- public int compareNonNull(String o1, String o2) {
- double d1 = Double.parseDouble(o1);
- double d2 = Double.parseDouble(o2);
- return Double.compare(d1, d2);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
deleted file mode 100644
index d599abc..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.common.RowKeySplitter;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowKeyDecoder {
-
- private final CubeDesc cubeDesc;
- private final RowKeyColumnIO colIO;
- private final RowKeySplitter rowKeySplitter;
-
- private Cuboid cuboid;
- private List<String> names;
- private List<String> values;
-
- public RowKeyDecoder(CubeSegment cubeSegment) {
- this.cubeDesc = cubeSegment.getCubeDesc();
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
- this.colIO = new RowKeyColumnIO(cubeSegment);
- this.values = new ArrayList<String>();
- }
-
- public long decode(byte[] bytes) throws IOException {
- this.values.clear();
-
- long cuboidId = rowKeySplitter.split(bytes, bytes.length);
- initCuboid(cuboidId);
-
- SplittedBytes[] splits = rowKeySplitter.getSplitBuffers();
-
- int offset = 1; // skip cuboid id part
-
- for (int i = 0; i < this.cuboid.getColumns().size(); i++) {
- TblColRef col = this.cuboid.getColumns().get(i);
- collectValue(col, splits[offset].value, splits[offset].length);
- offset++;
- }
-
- return cuboidId;
- }
-
- private void initCuboid(long cuboidID) {
- if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
- return;
- }
- this.cuboid = Cuboid.findById(cubeDesc, cuboidID);
- }
-
- private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {
- String strValue = colIO.readColumnString(col, valueBytes, length);
- values.add(strValue);
- }
-
- public RowKeySplitter getRowKeySplitter() {
- return rowKeySplitter;
- }
-
- public void setCuboid(Cuboid cuboid) {
- this.cuboid = cuboid;
- this.names = null;
- }
-
- public List<String> getNames(Map<TblColRef, String> aliasMap) {
- if (names == null) {
- names = buildNameList(aliasMap);
- }
- return names;
- }
-
- private List<String> buildNameList(Map<TblColRef, String> aliasMap) {
- List<TblColRef> columnList = getColumns();
- List<String> result = new ArrayList<String>(columnList.size());
- for (TblColRef col : columnList)
- result.add(findName(col, aliasMap));
- return result;
- }
-
- private String findName(TblColRef column, Map<TblColRef, String> aliasMap) {
- String name = null;
- if (aliasMap != null) {
- name = aliasMap.get(column);
- }
- if (name == null) {
- name = column.getName();
- }
- return name;
- }
-
- public List<TblColRef> getColumns() {
- return cuboid.getColumns();
- }
-
- public List<String> getValues() {
- return values;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(cuboid.getId());
- for (Object value : values) {
- buf.append(",");
- buf.append(value);
- }
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
deleted file mode 100644
index 90676ba..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author George Song (ysong1)
- */
-public class RowKeyEncoder extends AbstractRowKeyEncoder {
-
- private int bytesLength;
- protected int headerLength;
- private RowKeyColumnIO colIO;
-
- protected RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
- super(cuboid);
- colIO = new RowKeyColumnIO(cubeSeg);
- bytesLength = headerLength = RowConstants.ROWKEY_CUBOIDID_LEN; // header
- for (TblColRef column : cuboid.getColumns()) {
- bytesLength += colIO.getColumnLength(column);
- }
- }
-
- public RowKeyColumnIO getColumnIO() {
- return colIO;
- }
-
- public int getColumnOffset(TblColRef col) {
- int offset = RowConstants.ROWKEY_CUBOIDID_LEN;
-
- for (TblColRef dimCol : cuboid.getColumns()) {
- if (col.equals(dimCol))
- return offset;
- offset += colIO.getColumnLength(dimCol);
- }
-
- throw new IllegalArgumentException("Column " + col + " not found on cuboid " + cuboid);
- }
-
- public int getColumnLength(TblColRef col) {
- return colIO.getColumnLength(col);
- }
-
- public int getRowKeyLength() {
- return bytesLength;
- }
-
- public int getHeaderLength() {
- return headerLength;
- }
-
- @Override
- public byte[] encode(Map<TblColRef, String> valueMap) {
- List<byte[]> valueList = new ArrayList<byte[]>();
- for (TblColRef bdCol : cuboid.getColumns()) {
- String value = valueMap.get(bdCol);
- valueList.add(valueStringToBytes(value));
- }
- byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER);
- return encode(values);
- }
-
- public byte[] valueStringToBytes(String value) {
- if (value == null)
- return null;
- else
- return Bytes.toBytes(value);
- }
-
- @Override
- public byte[] encode(byte[][] values) {
- byte[] bytes = new byte[this.bytesLength];
- int offset = fillHeader(bytes, values);
-
- for (int i = 0; i < cuboid.getColumns().size(); i++) {
- TblColRef column = cuboid.getColumns().get(i);
- int colLength = colIO.getColumnLength(column);
- byte[] value = values[i];
- if (value == null) {
- fillColumnValue(column, colLength, null, 0, bytes, offset);
- } else {
- fillColumnValue(column, colLength, value, value.length, bytes, offset);
- }
- offset += colLength;
-
- }
- return bytes;
- }
-
- protected int fillHeader(byte[] bytes, byte[][] values) {
- int offset = 0;
- System.arraycopy(cuboid.getBytes(), 0, bytes, offset, RowConstants.ROWKEY_CUBOIDID_LEN);
- offset += RowConstants.ROWKEY_CUBOIDID_LEN;
- if (this.headerLength != offset) {
- throw new IllegalStateException("Expected header length is " + headerLength + ". But the offset is " + offset);
- }
- return offset;
- }
-
- protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
- // special null value case
- if (value == null) {
- byte[] valueBytes = defaultValue(columnLen);
- System.arraycopy(valueBytes, 0, outputValue, outputValueOffset, columnLen);
- return;
- }
-
- colIO.writeColumn(column, value, valueLen, this.blankByte, outputValue, outputValueOffset);
- }
-
- protected byte[] defaultValue(int length) {
- byte[] values = new byte[length];
- Arrays.fill(values, this.blankByte);
- return values;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java b/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
deleted file mode 100644
index 5fe4e2e..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/kv/RowValueDecoder.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.kv;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowValueDecoder implements Cloneable {
-
- private final HBaseColumnDesc hbaseColumn;
- private final MeasureCodec codec;
- private final BitSet projectionIndex;
- private final MeasureDesc[] measures;
- private final List<String> names;
- private Object[] values;
-
- public RowValueDecoder(RowValueDecoder rowValueDecoder) {
- this.hbaseColumn = rowValueDecoder.getHBaseColumn();
- this.projectionIndex = rowValueDecoder.getProjectionIndex();
- this.names = new ArrayList<String>();
- this.measures = hbaseColumn.getMeasures();
- for (MeasureDesc measure : measures) {
- this.names.add(measure.getFunction().getRewriteFieldName());
- }
- this.codec = new MeasureCodec(measures);
- this.values = new Object[measures.length];
- }
-
- public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
- this.hbaseColumn = hbaseColumn;
- this.projectionIndex = new BitSet();
- this.names = new ArrayList<String>();
- this.measures = hbaseColumn.getMeasures();
- for (MeasureDesc measure : measures) {
- this.names.add(measure.getFunction().getRewriteFieldName());
- }
- this.codec = new MeasureCodec(measures);
- this.values = new Object[measures.length];
- }
-
- public void decode(byte[] bytes) {
- codec.decode(ByteBuffer.wrap(bytes), values);
- convertToJavaObjects(values, values);
- }
-
- private void convertToJavaObjects(Object[] mapredObjs, Object[] results) {
- for (int i = 0; i < mapredObjs.length; i++) {
- Object o = mapredObjs[i];
-
- // if (o instanceof LongWritable)
- // o = ((LongWritable) o).get();
- // else if (o instanceof IntWritable)
- // o = ((IntWritable) o).get();
- // else if (o instanceof DoubleWritable)
- // o = ((DoubleWritable) o).get();
- // else if (o instanceof FloatWritable)
- // o = ((FloatWritable) o).get();
-
- if (o instanceof LongWritable)
- o = ((LongWritable) o).get();
- else if (o instanceof DoubleWritable)
- o = ((DoubleWritable) o).get();
-
- results[i] = o;
- }
- }
-
- public void setIndex(int bitIndex) {
- projectionIndex.set(bitIndex);
- }
-
- public HBaseColumnDesc getHBaseColumn() {
- return hbaseColumn;
- }
-
- public BitSet getProjectionIndex() {
- return projectionIndex;
- }
-
- public Object[] getValues() {
- return values;
- }
-
- public List<String> getNames() {
- return names;
- }
-
- public MeasureDesc[] getMeasures() {
- return measures;
- }
-
- public boolean hasMemHungryMeasures() {
- for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
- FunctionDesc func = measures[i].getFunction();
- if (func.getMeasureType().isMemoryHungry())
- return true;
- }
- return false;
- }
-
- public static boolean hasMemHungryMeasures(Collection<RowValueDecoder> rowValueDecoders) {
- for (RowValueDecoder decoder : rowValueDecoders) {
- if (decoder.hasMemHungryMeasures())
- return true;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
deleted file mode 100644
index 53c7135..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-/**
- * @author xduo
- *
- */
-public enum CubeBuildTypeEnum {
- /**
- * rebuild a segment or incremental build
- */
- BUILD,
- /**
- * merge segments
- */
- MERGE,
-
- /**
- * refresh segments
- */
- REFRESH
-}
[22/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
deleted file mode 100644
index 73abe26..0000000
--- a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
+++ /dev/null
@@ -1,8 +0,0 @@
-16,Consumer-Other,16,102,Consumer,2012-08-09,USER_P,,USER_A
-5,Occasional,5,102,Consumer,2010-05-11,USER_H,2011-03-01 11:30:52,USER_A
-15,Professional-Other,15,101,Professional,2012-08-09,USER_P,,USER_A
--99,Not Applicable,-99,-99,Not Applicable,2010-05-11,USER_H,,USER_A
-14,Regular,14,102,Consumer,2011-03-01,USER_H,,USER_A
-12,Merchant,12,101,Professional,2011-03-01,USER_H,,USER_A
-11,Large Merchant,11,101,Professional,2011-03-01,USER_H,,USER_A
-13,Entrepreneur,13,101,Professional,2011-03-01,USER_H,,USER_A
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/EDW.TEST_SITES.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SITES.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SITES.csv
deleted file mode 100644
index 2829050..0000000
--- a/examples/test_case_data/localmeta/data/EDW.TEST_SITES.csv
+++ /dev/null
@@ -1,262 +0,0 @@
-248,Uruguay,S,1,0,211,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-166,Qatar,S,1,0,166,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-84,Guam,S,1,0,84,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-225,Peru,S,45,0,161,2005-04-27,2013-11-27 00:14:36,USER_X,USER_X
-103,Jan Mayen,S,1,0,103,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-63,El Salvador,S,1,0,63,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-124,Macedonia,S,1,0,124,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-147,Netherlands Antilles,S,1,0,147,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-21,Barbados,S,1,0,21,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-82,Grenada,S,1,0,82,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-980,CoBrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-2,Canada,S,2,1,2,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-204,Indonesia,S,24,0,96,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-246,Tuvalu,S,1,0,207,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-25,Benin,S,1,0,25,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
--6,StubHub US,S,1,1,1,2012-11-04,,USER_C4,USER_X
-86,Guernsey,S,1,0,86,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-187,Sri Lanka,S,1,0,187,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-6,Algeria,S,1,0,6,2000-07-26,2013-11-27 00:14:36,USER_G,USER_X
-984,Digit City Cobrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-145,Nepal,S,1,0,145,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-4,Afghanistan,S,1,0,4,2000-07-26,2013-11-27 00:14:36,USER_G,USER_X
-227,Iceland,S,1,0,94,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-229,Serbia,S,1,0,229,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-65,Eritrea,S,1,0,65,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-44,Chile,S,1,0,44,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-170,Saint Helena,S,1,0,170,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-88,Guinea-Bissau,S,1,0,88,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-46,Colombia,S,1,0,46,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
--4,Gmarket,S,48,0,111,2010-05-23,,USER_X,USER_X
-69,Fiji,S,1,0,69,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-48,Congo Democratic Republic Of The,S,1,0,48,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-231,Turkey,S,1,0,204,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-126,Malawi,S,1,0,126,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
--888,Invalid,S,-999,0,-999,2004-12-03,,USER_X,USER_X
-107,Kazakhstan,S,1,0,107,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-982,Netscape Cobrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-900,Gbh,S,1,0,1,2012-10-25,2013-11-27 00:14:36,USER_X,USER_X
-208,New Zealand,S,33,0,149,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-71,France,S,7,1,71,2000-09-06,2013-11-27 00:14:36,USER_S,USER_X
-10,Anguilla,S,1,0,10,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-172,Saint Lucia,S,1,0,172,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-153,Niue,S,1,0,153,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-67,Ethiopia,S,1,0,67,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-128,Maldives,S,1,0,128,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-233,United Arab Emirates,S,1,0,210,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-143,Namibia,S,1,0,143,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-115,Latvia,S,1,0,115,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-206,Israel,S,26,0,100,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-216,Singapore,S,39,0,180,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-105,Jersey,S,1,0,105,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-159,Papua New Guinea,S,1,0,159,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-220,Vietnam,S,47,0,216,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-241,Tonga,S,1,0,201,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-201,Hong Kong,S,22,0,92,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-117,Lesotho,S,1,0,117,2013-02-22,2013-11-27 00:14:36,USER_X,USER_X
-258,Zambia,S,1,0,223,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-237,Reunion,S,1,0,227,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-239,Tanzania,S,1,0,198,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-971,Butterfields,D,1,0,-999,2000-08-30,,USER_G,USER_X
-136,Mexico,S,1,0,136,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-50,Cook Islands,S,1,0,50,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-193,Switzerland,S,13,0,193,2002-08-30,2013-11-27 00:14:36,USER_S,USER_X
-989,AutoTrader,D,1,0,-999,2000-06-29,,USER_G,USER_X
-16,Austria,S,7,0,16,2002-06-11,2013-11-27 00:14:36,USER_S,USER_X
-30,Botswana,S,1,0,30,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-178,Seychelles,S,1,0,178,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-152,Nigeria,S,1,0,152,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-9,Angola,S,1,0,9,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-23,Belgium (French),S,7,1,23,2001-12-27,2013-11-27 00:14:36,USER_S,USER_X
-987,Card,D,1,0,-999,2000-06-29,,USER_G,USER_X
-91,Honduras,S,1,0,91,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-232,Ukraine,S,1,0,209,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-89,Guyana,S,1,0,89,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-129,Mali,S,1,0,129,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-234,Bangladesh,S,1,0,20,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
--5,Gitti gidiyor,S,49,0,204,2012-11-07,,USER_C4,USER_X
-26,Bermuda,S,1,0,26,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-49,Congo Republic Of The,S,1,0,49,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-148,New Caledonia,S,1,0,148,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-66,Estonia,S,1,0,66,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-5,Albania,S,1,0,5,2000-07-26,2013-11-27 00:14:36,USER_G,USER_X
-108,Kenya,S,1,0,108,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-983,ICQ,D,1,0,-999,2000-07-07,,USER_PFI,USER_X
-167,Romania,S,1,0,167,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-169,Rwanda,S,1,0,169,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-228,Montenegro,S,1,0,228,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-106,Jordan,S,1,0,106,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-213,Purtugal,S,7,0,164,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-207,Malaysia,S,46,0,127,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-68,Falkland Islands (Islas Makvinas),S,1,0,68,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-247,Uganda,S,1,0,208,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-125,Madagascar,S,1,0,125,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-43,Chad,S,1,0,43,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-219,Thailand,S,40,0,199,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-123,Belgium (Dutch),S,7,1,23,2001-12-27,2013-11-27 00:14:36,USER_S,USER_X
-15,Australia,S,5,1,15,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-144,Nauru,S,1,0,144,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-81,Greenland,S,1,0,81,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-142,Mozambique,S,1,0,142,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-104,Japan,S,6,0,104,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-102,Jamaica,S,1,0,102,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-186,Spain,S,7,0,186,2001-10-20,2013-11-27 00:14:36,USER_S,USER_X
-100,Ebaymotors,S,1,0,1,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-116,Lebanon South,S,1,0,116,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-76,Georgia,S,1,0,76,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-137,Moldova,S,1,0,137,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-79,Gibraltar,S,1,0,79,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-87,Guinea,S,1,0,87,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-996,Stamps,D,1,0,-999,2000-06-29,,USER_G,USER_X
-160,Paraguay,S,1,0,160,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-240,Togo,S,1,0,200,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-203,India,S,44,0,95,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-3,英国,S,3,1,3,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-198,Denmark,S,17,0,57,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-37,Burundi,S,1,0,37,2001-09-07,2013-11-27 00:14:36,USER_DNEL,USER_X
-20,Ebay Half Integration,S,1,0,1,2002-03-21,2013-11-27 00:14:36,USER_S,USER_X
-55,Cyprus,S,1,0,55,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-259,Zimbabwe,S,1,0,224,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-120,Liechtenstein,S,1,0,120,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-156,Pakistan,S,1,0,156,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-205,Ireland,S,7,0,99,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-998,Computers,D,1,0,-999,2000-06-29,,USER_G,USER_X
-157,Palau,S,1,0,157,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-182,Slovenia,S,1,0,182,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-35,Burkina Faso,S,1,0,35,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-199,Finland,S,7,0,70,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-197,Czech Republic,S,16,0,56,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-138,Monaco,S,1,0,138,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-985,AOL Cobrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-75,Gambia,S,1,0,75,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-14,Aruba,S,1,0,14,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-218,Sweden,S,38,0,192,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-24,Belize,S,1,0,24,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-217,South Africa,S,43,0,185,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-970,Butterfields Kiss,D,1,0,-999,2000-08-30,,USER_G,USER_X
-17,Azerbaijan Republic,S,1,0,17,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-993,Percelain,D,1,0,-999,2000-06-12,,USER_G,USER_X
-211,Philippines,S,34,0,162,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-155,Oman,S,1,0,155,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-73,French Polynesia,S,1,0,73,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-53,Croatia Democratic Republic Of The,S,1,0,53,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-134,Mauritius,S,1,0,134,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-113,Kyrgyzstan,S,1,0,113,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-131,Marshall Islands,S,1,0,131,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-992,Jewelry,D,1,0,-999,2000-06-29,,USER_G,USER_X
-174,Saint Vincent And The Grenadines,S,1,0,174,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-22,Belarus,S,1,0,22,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-256,Western Samoa,S,1,0,220,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-52,Cote D Ivoire (Ivory Coast),S,1,0,52,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-175,San Marino,S,1,0,175,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
--3,Tradera,S,7,0,192,2009-02-11,,USER_OFF,USER_X
-200,Greece,S,7,0,80,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-32,British Virgin Islands,S,1,0,32,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-171,Saint Kitts-Nevis,S,1,0,171,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-176,Saudi Arabia,S,1,0,176,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-19,Bahrain,S,1,0,19,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-101,Italy,S,7,1,101,2000-11-03,2013-11-27 00:14:36,USER_S,USER_X
-236,Micronesia,S,1,0,226,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-64,Equatorial Guinea,S,1,0,64,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-209,Norway,S,32,0,154,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-195,Tahiti,S,1,0,195,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-221,Us1,S,1,0,1,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-146,Netherlands,S,7,1,146,2001-07-12,2013-11-27 00:14:36,USER_S,USER_X
-214,Puerto Rico,S,1,0,165,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-29,Bosnia And Herzegovina,S,1,0,29,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-215,Russia,S,84,0,168,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-90,Haiti,S,1,0,90,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-12,Argentina,S,1,0,12,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-158,Panama,S,1,0,158,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-139,Mongolia,S,1,0,139,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-41,Cayman Islands,S,1,0,41,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-151,Niger,S,1,0,151,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-11,Antigua And Barbuda,S,1,0,11,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
--1,Korea Auction.co.kr,S,48,0,111,2009-02-11,,USER_OFF,USER_X
-112,Kuwait,S,1,0,112,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-59,Dominica,S,1,0,59,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-179,Sierra Leone,S,1,0,179,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-132,Martinique,S,1,0,132,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-972,Hoianhoard,D,1,0,-999,2000-07-26,,USER_G,USER_X
-257,Yemen,S,1,0,221,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-235,Macau,S,1,0,123,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-8,Andorra,S,1,0,8,2000-07-26,2013-11-27 00:14:36,USER_G,USER_X
-191,Swaziland,S,1,0,191,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-141,Morocco,S,1,0,141,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-177,Senegal,S,1,0,177,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-184,Somalia,S,1,0,184,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-47,Comoros,S,1,0,47,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-212,Poland,S,35,0,163,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-994,Sports,D,1,0,-999,2000-07-07,,USER_PFI,USER_X
-34,Bulgaria,S,1,0,34,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
--2,Bazee,S,44,0,95,2009-02-11,,USER_OFF,USER_X
-224,Japan Row,S,6,0,104,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
--999,Unknown,S,-999,0,-999,2000-06-29,,USER_G,USER_X
-986,GO Cobrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-133,Mauritania,S,1,0,133,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-196,Taiwan,S,41,0,196,2002-03-08,2013-11-27 00:14:36,USER_S,USER_X
-33,Brunei Darussalam,S,1,0,33,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
--7,StubHub UK,S,3,1,3,2012-11-08,,USER_C4,USER_X
-140,Montserrat,S,1,0,140,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-990,Toys,D,1,0,-999,2000-06-29,,USER_G,USER_X
-150,Nicaragua,S,1,0,150,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-244,Turkmenistan,S,1,0,205,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-78,Ghana,S,1,0,78,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-973,Developer.ebay.com,D,1,0,-999,2000-10-06,2000-10-06 00:00:00,USER_G,USER_G
-202,Hungary,S,23,0,93,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-242,Trinidad And Tobago,S,1,0,202,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-981,Compuserve Cobrand,D,1,0,-999,2000-06-29,,USER_G,USER_X
-60,Dominican Republic,S,1,0,60,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-13,Armenia,S,1,0,13,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-991,Dolls,D,1,0,-999,2000-06-29,,USER_G,USER_X
-118,Liberia,S,1,0,118,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-77,Germany,S,7,1,77,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-51,Costa Rica,S,1,0,51,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-1,Usa,S,1,1,1,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
-39,Cameroon,S,1,0,39,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-997,Great Collections,D,1,0,-999,2000-06-29,,USER_G,USER_X
-181,Slovak Republic,S,1,0,181,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-189,Suriname,S,1,0,189,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-74,Gabon Republic,S,1,0,74,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-31,Brazil,S,1,0,31,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-38,Cambodia,S,1,0,38,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-245,Turks And Caicos Islands,S,1,0,206,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-28,Bolivia,S,1,0,28,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-210,Canada (French),S,2,0,2,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-238,Tajikistan,S,1,0,197,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-58,Djibouti,S,1,0,58,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-135,Mayotte,S,1,0,135,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-995,Coins,D,1,0,-999,2000-06-29,,USER_G,USER_X
-114,Laos,S,1,0,114,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-173,Saint Pierre And Miquelon,S,1,0,173,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-83,Guadeloupe,S,1,0,83,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-62,Egypt,S,1,0,62,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-122,Luxembourg,S,1,0,122,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-72,French Guiana,S,1,0,72,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-223,China,S,14,0,45,2003-10-04,2013-11-27 00:14:36,USER_X,USER_X
-163,Unknown: Added by DW(0),S,-999,0,-999,2001-09-07,,USER_DNEL,USER_X
-40,Cape Verde Islands,S,1,0,40,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-222,Yugoslavia,S,1,0,222,2013-03-12,2013-11-27 00:14:36,USER_X,USER_X
-61,Ecuador,S,1,0,61,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-243,Tunisia,S,1,0,203,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-183,Solomon Islands,S,1,0,183,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-226,South Korea,S,1,0,111,2012-04-27,2013-11-27 00:14:36,USER_X,USER_X
-7,American Samoa,S,1,0,7,2000-07-26,2013-11-27 00:14:36,USER_G,USER_X
-121,Lithuania,S,1,0,121,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-18,Bahamas,S,1,0,18,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-0,Ebay,S,1,0,1,2000-06-09,2013-11-27 00:14:36,USER_S,USER_X
--8,Wholesale Deals -- US,S,1,1,1,2013-02-19,,USER_YAJ,USER_X
-250,Vanuatu,S,1,0,213,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-42,Central African Republic,S,1,0,42,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-252,Venezuela,S,1,0,215,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-109,Kiribati,S,1,0,109,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-130,Malta,S,1,0,130,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-251,Vatican City State,S,1,0,214,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-253,Virgin Islands (U.S.),S,1,0,217,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-500,Unknown: Added by DW(1),S,-999,0,-999,2000-07-26,,USER_G,USER_X
-255,Western Sahara,S,1,0,219,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-190,Svalbard,S,1,0,190,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-85,Guatemala,S,1,0,85,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-254,Wallis And Futuna,S,1,0,218,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
-27,Bhutan,S,1,0,27,2013-01-29,2013-11-27 00:14:36,USER_X,USER_X
-249,Uzbekistan,S,1,0,212,2013-01-30,2013-11-27 00:14:36,USER_X,USER_X
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
deleted file mode 100644
index 2da72b7..0000000
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "columnConfigs": [
- {
- "columnName": "lstg_format_name",
- "valueSet": [
- "FP-GTC",
- "FP-non GTC",
- "ABIN",
- "Auction",
- "Others"
- ],
- "exclusive": true
- },
- {
- "columnName": "SELLER_ID",
- "valueSet": [
- "10000000",
- "10001000"
- ],
- "asRange": true
- },
- {
- "columnName": "ITEM_COUNT",
- "valueSet": [
- "0",
- "2000000"
- ],
- "asRange": true
- }
- ]
-}
[41/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java b/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
deleted file mode 100644
index 63e412f..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/InstallJarIntoMavenTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.junit.Ignore;
-
-/**
- * Created by honma on 6/6/14.
- */
-public class InstallJarIntoMavenTest {
-
- @Ignore("convenient trial tool for dev")
- public void testInstall() throws IOException {
- File folder = new File("/export/home/b_kylin/tmp");
- File out = new File("/export/home/b_kylin/tmp/out.sh");
- out.createNewFile();
- FileWriter fw = new FileWriter(out);
-
- for (File file : folder.listFiles()) {
- String name = file.getName();
-
- if (!name.endsWith(".jar"))
- continue;
-
- int firstSlash = name.indexOf('-');
- int lastDot = name.lastIndexOf('.');
- String groupId = name.substring(0, firstSlash);
-
- Pattern pattern = Pattern.compile("-\\d");
- Matcher match = pattern.matcher(name);
- match.find();
- String artifactId = name.substring(0, match.start());
- String version = name.substring(match.start() + 1, lastDot);
-
- fw.write(String.format("mvn install:install-file -Dfile=%s -DgroupId=%s -DartifactId=%s -Dversion=%s -Dpackaging=jar", name, "org.apache." + groupId, artifactId, version));
- fw.write("\n");
- }
- fw.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
deleted file mode 100644
index 1777923..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-
-/**
- * @author ysong1
- */
-public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
-
- @Override
- public void createTestMetadata() {
- createTestMetadata(LOCALMETA_TEST_DATA);
- }
-
- public static void createTestMetadata(String testDataFolder) {
- KylinConfig.destoryInstance();
-
- String tempTestMetadataUrl = "../examples/test_metadata";
- try {
- FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
- FileUtils.copyDirectory(new File(testDataFolder), new File(tempTestMetadataUrl));
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, tempTestMetadataUrl);
-
- KylinConfig.getInstanceFromEnv().setMetadataUrl(tempTestMetadataUrl);
- }
-
- public static void cleanAfterClass() {
- String tempTestMetadataUrl = "../examples/test_metadata";
- try {
- FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
- } catch (IOException e) {
- throw new IllegalStateException("Can't delete directory " + tempTestMetadataUrl, e);
- }
- staticCleanupTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- cleanAfterClass();
- }
-
- protected ResourceStore getStore() {
- return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java b/common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
deleted file mode 100644
index dabf238..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-@Ignore("convenient trial tool for dev")
-public class MailServiceTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
-
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testSendEmail() throws IOException {
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- MailService mailservice = new MailService(config);
- boolean sent = sendTestEmail(mailservice);
- assert sent;
-
- // set mail.enabled=false, and run again, this time should be no mail delviered
- config.setProperty(KylinConfig.MAIL_ENABLED, "false");
- mailservice = new MailService(config);
- sent = sendTestEmail(mailservice);
- assert !sent;
-
- }
-
- private boolean sendTestEmail(MailService mailservice) {
-
- List<String> receivers = new ArrayList<String>(1);
- receivers.add("foobar@foobar.com");
- try {
- return mailservice.sendMail(receivers, "A test email from Kylin", "Hello!");
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java b/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
deleted file mode 100644
index b603100..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/PartialSorterTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import static org.junit.Assert.assertArrayEquals;
-
-import java.util.Comparator;
-import java.util.List;
-
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
- */
-public class PartialSorterTest {
- @Test
- public void basicTest() {
- List<Integer> a = Lists.newArrayList();
- a.add(100);
- a.add(2);
- a.add(92);
- a.add(1);
- a.add(0);
- PartialSorter.partialSort(a, Lists.newArrayList(1, 3, 4), new Comparator<Integer>() {
- @Override
- public int compare(Integer o1, Integer o2) {
- return o1.compareTo(o2);
- }
- });
- assertArrayEquals(a.toArray(), new Integer[] { 100, 0, 92, 1, 2 });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/RandomSamplerTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/RandomSamplerTest.java b/common/src/test/java/org/apache/kylin/common/util/RandomSamplerTest.java
deleted file mode 100644
index 67f4e1d..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/RandomSamplerTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RandomSamplerTest {
-
- @Test
- public void test() {
- RandomSampler<Text> s = new RandomSampler<Text>();
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1000; i++) {
- data.add(new Text(String.valueOf(i)));
- }
-
- List<Text> result = s.sample(data, 50);
- System.out.println(result);
- assertEquals(50, result.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java b/common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
deleted file mode 100644
index 1141831..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class SSHClientTest extends LocalFileMetadataTestCase {
-
- private boolean isRemote;
- private String hostname;
- private String username;
- private String password;
-
- private void loadPropertiesFile() throws IOException {
-
- KylinConfig cfg = KylinConfig.getInstanceFromEnv();
-
- this.isRemote = cfg.getRunAsRemoteCommand();
- this.hostname = cfg.getRemoteHadoopCliHostname();
- this.username = cfg.getRemoteHadoopCliUsername();
- this.password = cfg.getRemoteHadoopCliPassword();
- }
-
- @Before
- public void before() throws Exception {
- this.createTestMetadata();
- loadPropertiesFile();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCmd() throws Exception {
- if (isRemote == false)
- return;
-
- SSHClient ssh = new SSHClient(this.hostname, this.username, this.password);
- SSHClientOutput output = ssh.execCommand("echo hello");
- assertEquals(0, output.getExitCode());
- assertEquals("hello\n", output.getText());
- }
-
- @Test
- public void testScp() throws Exception {
- if (isRemote == false)
- return;
-
- SSHClient ssh = new SSHClient(this.hostname, this.username, this.password);
- File tmpFile = FileUtil.createLocalTempFile(new File("/tmp/test_scp"), "temp_", false);
- ssh.scpFileToRemote(tmpFile.getAbsolutePath(), "/tmp");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
deleted file mode 100644
index e0727ed..0000000
--- a/conf/kylin.properties
+++ /dev/null
@@ -1,149 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=localhost:7070
-
-#set display timezone on UI,format like[GMT+N or GMT-N]
-kylin.rest.timezone=GMT-8
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-# HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020
-# leave empty if hbase running on same cluster with hive and mapreduce
-kylin.hbase.cluster.fs=
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-kylin.server.mode=all
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-# It should not be set to "true" unless you're NOT running Kylin.sh on a hadoop client machine
-# (Thus kylin instance has to ssh to another real hadoop client machine to execute hbase,hive,hadoop commands)
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-#if you should getting job status from RM with kerberos, set it true..
-kylin.job.status.with.kerberos=false
-
-# Hive database name for putting the intermediate flat tables
-kylin.job.hive.database.for.intermediatetable=default
-
-#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=snappy
-
-# The cut size for hbase region, in GB.
-# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
-kylin.hbase.region.cut.small=10
-kylin.hbase.region.cut.medium=20
-kylin.hbase.region.cut.large=100
-
-# HBase min and max region count
-kylin.hbase.region.count.min=1
-kylin.hbase.region.count.max=500
-
-# Enable/disable ACL check for cube query
-kylin.query.security.enabled=true
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-## Config for mail service
-
-# If true, will send email notification;
-mail.enabled=false
-mail.host=
-mail.username=
-mail.password=
-mail.sender=
-
-###########################config info for web#######################
-
-#help info ,format{name|displayName|link} ,optional
-kylin.web.help.length=4
-kylin.web.help.0=start|Getting Started|
-kylin.web.help.1=odbc|ODBC Driver|
-kylin.web.help.2=tableau|Tableau Guide|
-kylin.web.help.3=onboard|Cube Design Tutorial|
-#hadoop url link ,optional
-kylin.web.hadoop=
-#job diagnostic url link ,optional
-kylin.web.diagnostic=
-#contact mail on web page ,optional
-kylin.web.contact_mail=
-#set the number of databases and tables of each database shown on the hive tree
-kylin.web.hive.limit=20
-###########################config info for front#######################
-
-#env DEV|QA|PROD
-deploy.env=DEV
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
-
-
-###########################config info for kylin monitor#######################
-# hive jdbc url
-kylin.monitor.hive.jdbc.connection.url=
-
-#config where to parse query log,split with comma ,will also read $KYLIN_HOME/tomcat/logs/ by default
-kylin.monitor.ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2
-
-#will create external hive table to query result csv file
-#will set to kylin_query_log by default if not config here
-kylin.monitor.query.log.parse.result.table = kylin_query_log
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/conf/kylin_job_conf.xml b/conf/kylin_job_conf.xml
deleted file mode 100644
index 2276a7c..0000000
--- a/conf/kylin_job_conf.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-
-
- <property>
- <name>hive.merge.mapfiles</name>
- <value>true</value>
- <description>Enable hive file merge on mapper only job</description>
- </property>
- <property>
- <name>hive.merge.mapredfiles</name>
- <value>true</value>
- <description>Enable hive file merge on map-reduce job</description>
- </property>
- <property>
- <name>hive.merge.size.per.task</name>
- <value>268435456</value>
- <description>Size for the merged file: 256M</description>
- </property>
-
- <property>
- <name>hive.support.concurrency</name>
- <value>false</value>
- <description>Hive concurrency lock</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/cube/.settings/org.eclipse.core.resources.prefs b/cube/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 04cfa2c..0000000
--- a/cube/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/cube/.settings/org.eclipse.jdt.core.prefs b/cube/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a903301..0000000
--- a/cube/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/cube/.settings/org.eclipse.jdt.ui.prefs b/cube/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index dece0e6..0000000
--- a/cube/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/pom.xml
----------------------------------------------------------------------
diff --git a/cube/pom.xml b/cube/pom.xml
deleted file mode 100644
index b4fa9da..0000000
--- a/cube/pom.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-cube</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:Cube</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.parent.version}</version>
- </dependency>
-
- <!--Kylin Jar -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-metadata</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-dictionary</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.ning</groupId>
- <artifactId>compress-lzf</artifactId>
- </dependency>
- <dependency>
- <groupId>org.roaringbitmap</groupId>
- <artifactId>RoaringBitmap</artifactId>
- </dependency>
- <!--<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
- </dependency>-->
-
- <!-- Env & Test -->
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <classifier>hadoop2</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- <!-- version conflict with hadoop2.2 -->
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <!--
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>jar-with-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/assemble.xml</descriptor>
- </descriptors>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- -->
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/assembly/assemble.xml
----------------------------------------------------------------------
diff --git a/cube/src/main/assembly/assemble.xml b/cube/src/main/assembly/assemble.xml
deleted file mode 100644
index 2596603..0000000
--- a/cube/src/main/assembly/assemble.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>jar-with-dependencies</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <useProjectArtifact>true</useProjectArtifact>
- <unpack>true</unpack>
- <scope>runtime</scope>
- <excludes>
- <exclude>javax.servlet*:*</exclude>
- </excludes>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
deleted file mode 100644
index 5668030..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Sets;
-
-/**
- */
-public class CubeCapabilityChecker {
- private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class);
-
- public static CapabilityResult check(CubeInstance cube, SQLDigest digest) {
- CapabilityResult result = new CapabilityResult();
- result.capable = false;
-
- // match joins
- boolean isJoinMatch = isJoinMatch(digest.joinDescs, cube);
- if (!isJoinMatch) {
- logger.info("Exclude cube " + cube.getName() + " because unmatched joins");
- return result;
- }
-
- // dimensions & measures
- Collection<TblColRef> dimensionColumns = getDimensionColumns(digest);
- Collection<FunctionDesc> aggrFunctions = digest.aggregations;
- Collection<TblColRef> unmatchedDimensions = unmatchedDimensions(dimensionColumns, cube);
- Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube);
-
- // try custom measure types
- if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) {
- tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
- }
-
- // try dimension-as-measure
- if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result);
- }
-
- if (!unmatchedDimensions.isEmpty()) {
- logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions");
- return result;
- }
-
- if (!unmatchedAggregations.isEmpty()) {
- logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations");
- return result;
- }
-
- // cost will be minded by caller
- result.capable = true;
- return result;
- }
-
- private static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) {
- Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns;
- Collection<TblColRef> filterColumns = sqlDigest.filterColumns;
-
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-
- private static Set<TblColRef> unmatchedDimensions(Collection<TblColRef> dimensionColumns, CubeInstance cube) {
- HashSet<TblColRef> result = Sets.newHashSet(dimensionColumns);
- CubeDesc cubeDesc = cube.getDescriptor();
- result.removeAll(cubeDesc.listDimensionColumnsIncludingDerived());
- return result;
- }
-
- private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, CubeInstance cube) {
- HashSet<FunctionDesc> result = Sets.newHashSet(aggregations);
- CubeDesc cubeDesc = cube.getDescriptor();
- result.removeAll(cubeDesc.listAllFunctions());
- return result;
- }
-
- private static boolean isJoinMatch(Collection<JoinDesc> joins, CubeInstance cube) {
- CubeDesc cubeDesc = cube.getDescriptor();
-
- List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
- for (DimensionDesc d : cubeDesc.getDimensions()) {
- if (d.getJoin() != null) {
- cubeJoins.add(d.getJoin());
- }
- }
- for (JoinDesc j : joins) {
- // optiq engine can't decide which one is fk or pk
- String pTable = j.getPrimaryKeyColumns()[0].getTable();
- String factTable = cubeDesc.getFactTable();
- if (factTable.equals(pTable)) {
- j.swapPKFK();
- }
-
- // check primary key, all PK column should refer to same tale, the Fact Table of cube.
- // Using first column's table name to check.
- String fTable = j.getForeignKeyColumns()[0].getTable();
- if (!factTable.equals(fTable)) {
- logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
- return false;
- }
-
- // The hashcode() function of JoinDesc has been overwritten,
- // which takes into consideration: pk,fk,jointype
- if (!cubeJoins.contains(j)) {
- logger.info("Query joins don't macth on cube " + cube.getName());
- return false;
- }
- }
- return true;
- }
-
- private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
-
- Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
- while (it.hasNext()) {
- FunctionDesc functionDesc = it.next();
-
- if (cubeFuncs.contains(functionDesc)) {
- it.remove();
- continue;
- }
-
- // let calcite handle count
- if (functionDesc.isCount()) {
- it.remove();
- continue;
- }
-
- // calcite can do aggregation from columns on-the-fly
- List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
- if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) {
- result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
- it.remove();
- continue;
- }
- }
- }
-
- // custom measure types can cover unmatched dimensions or measures
- private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
- CubeDesc cubeDesc = cube.getDescriptor();
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
- break;
-
- MeasureType<?> measureType = measure.getFunction().getMeasureType();
- if (measureType instanceof BasicMeasureType)
- continue;
-
- CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure);
- if (inf != null)
- result.influences.add(inf);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
deleted file mode 100644
index 89f8b01..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Manager class for CubeDesc; extracted from #CubeManager
- * @author shaoshi
- *
- */
-public class CubeDescManager {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeDescManager.class);
-
- public static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
-
- // ============================================================================
-
- private KylinConfig config;
- // name ==> CubeDesc
- private CaseInsensitiveStringCache<CubeDesc> cubeDescMap = new CaseInsensitiveStringCache<CubeDesc>(Broadcaster.TYPE.CUBE_DESC);
-
- public static CubeDescManager getInstance(KylinConfig config) {
- CubeDescManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeDescManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeDescManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeDescManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- Cuboid.clearCache();
- }
-
- private CubeDescManager(KylinConfig config) throws IOException {
- logger.info("Initializing CubeDescManager with config " + config);
- this.config = config;
- reloadAllCubeDesc();
- }
-
- public CubeDesc getCubeDesc(String name) {
- return cubeDescMap.get(name);
- }
-
- /**
- * Reload CubeDesc from resource store It will be triggered by an desc
- * update event.
- *
- * @param name
- * @throws IOException
- */
- public CubeDesc reloadCubeDesc(String name) throws IOException {
-
- // Save Source
- String path = CubeDesc.getCubeDescResourcePath(name);
-
- // Reload the CubeDesc
- CubeDesc ndesc = loadCubeDesc(path);
-
- // Here replace the old one
- cubeDescMap.putLocal(ndesc.getName(), ndesc);
- Cuboid.reloadCache(ndesc.getName());
- return ndesc;
- }
-
- private CubeDesc loadCubeDesc(String path) throws IOException {
- ResourceStore store = getStore();
- CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("CubeDesc name must not be blank");
- }
-
- ndesc.init(config, getMetadataManager().getAllTablesMap());
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
- }
-
- return ndesc;
- }
-
- /**
- * Create a new CubeDesc
- *
- * @param cubeDesc
- * @return
- * @throws IOException
- */
- public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException {
- if (cubeDesc.getUuid() == null || cubeDesc.getName() == null)
- throw new IllegalArgumentException();
- if (cubeDescMap.containsKey(cubeDesc.getName()))
- throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists");
-
- try {
- cubeDesc.init(config, getMetadataManager().getAllTablesMap());
- } catch (IllegalStateException e) {
- cubeDesc.addError(e.getMessage(), true);
- }
- // Check base validation
- if (!cubeDesc.getError().isEmpty()) {
- return cubeDesc;
- }
- // Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
- ValidateContext context = validator.validate(cubeDesc, true);
- if (!context.ifPass()) {
- return cubeDesc;
- }
-
- cubeDesc.setSignature(cubeDesc.calculateSignature());
-
- String path = cubeDesc.getResourcePath();
- getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER);
- cubeDescMap.put(cubeDesc.getName(), cubeDesc);
-
- return cubeDesc;
- }
-
- // remove cubeDesc
- public void removeCubeDesc(CubeDesc cubeDesc) throws IOException {
- String path = cubeDesc.getResourcePath();
- getStore().deleteResource(path);
- cubeDescMap.remove(cubeDesc.getName());
- Cuboid.reloadCache(cubeDesc.getName());
- }
-
- // remove cubeDesc
- public void removeLocalCubeDesc(String name) throws IOException {
- cubeDescMap.removeLocal(name);
- Cuboid.reloadCache(name);
- }
-
- private void reloadAllCubeDesc() throws IOException {
- ResourceStore store = getStore();
- logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-
- cubeDescMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- CubeDesc desc;
- try {
- desc = loadCubeDesc(path);
- } catch (Exception e) {
- logger.error("Error loading cube desc " + path, e);
- continue;
- }
- if (path.equals(desc.getResourcePath()) == false) {
- logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath());
- continue;
- }
- if (cubeDescMap.containsKey(desc.getName())) {
- logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path);
- continue;
- }
-
- cubeDescMap.putLocal(desc.getName(), desc);
- }
-
- logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)");
- }
-
- /**
- * Update CubeDesc with the input. Broadcast the event into cluster
- *
- * @param desc
- * @return
- * @throws IOException
- */
- public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
- // Validate CubeDesc
- if (desc.getUuid() == null || desc.getName() == null) {
- throw new IllegalArgumentException();
- }
- String name = desc.getName();
- if (!cubeDescMap.containsKey(name)) {
- throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist.");
- }
-
- try {
- desc.init(config, getMetadataManager().getAllTablesMap());
- } catch (IllegalStateException e) {
- desc.addError(e.getMessage(), true);
- return desc;
- } catch (IllegalArgumentException e) {
- desc.addError(e.getMessage(), true);
- return desc;
- }
-
- // Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
- ValidateContext context = validator.validate(desc, true);
- if (!context.ifPass()) {
- return desc;
- }
-
- desc.setSignature(desc.calculateSignature());
-
- // Save Source
- String path = desc.getResourcePath();
- getStore().putResource(path, desc, CUBE_DESC_SERIALIZER);
-
- // Reload the CubeDesc
- CubeDesc ndesc = loadCubeDesc(path);
- // Here replace the old one
- cubeDescMap.put(ndesc.getName(), desc);
-
- return ndesc;
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private CubeManager getCubeManager() {
- return CubeManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
-}
[42/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
deleted file mode 100644
index 027045a..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.LoggerFactory;
-
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
-
-public class SSHClient {
- protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(SSHClient.class);
-
- private String hostname;
- private String username;
- private String password;
- private String identityPath;
-
- public SSHClient(String hostname, String username, String password) {
- this.hostname = hostname;
- this.username = username;
- if (password != null && new File(password).exists()) {
- this.identityPath = new File(password).getAbsolutePath();
- this.password = null;
- } else {
- this.password = password;
- this.identityPath = null;
- }
- }
-
- public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
- FileInputStream fis = null;
- try {
- System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
-
- Session session = newJSchSession();
- session.connect();
-
- boolean ptimestamp = false;
-
- // exec 'scp -t rfile' remotely
- String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- File _lfile = new File(localFile);
-
- if (ptimestamp) {
- command = "T " + (_lfile.lastModified() / 1000) + " 0";
- // The access time should be sent here,
- // but it is not accessible with JavaAPI ;-<
- command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- }
-
- // send "C0644 filesize filename", where filename should not include '/'
- long filesize = _lfile.length();
- command = "C0644 " + filesize + " ";
- if (localFile.lastIndexOf("/") > 0) {
- command += localFile.substring(localFile.lastIndexOf("/") + 1);
- } else if (localFile.lastIndexOf(File.separator) > 0) {
- command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
- } else {
- command += localFile;
- }
- command += "\n";
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
-
- // send a content of lfile
- fis = new FileInputStream(localFile);
- byte[] buf = new byte[1024];
- while (true) {
- int len = fis.read(buf, 0, buf.length);
- if (len <= 0)
- break;
- out.write(buf, 0, len); // out.flush();
- }
- fis.close();
- fis = null;
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- out.close();
-
- channel.disconnect();
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fis != null)
- fis.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public void scpFileToLocal(String rfile, String lfile) throws Exception {
- FileOutputStream fos = null;
- try {
- System.out.println("SCP remote file " + rfile + " to local " + lfile);
-
- String prefix = null;
- if (new File(lfile).isDirectory()) {
- prefix = lfile + File.separator;
- }
-
- Session session = newJSchSession();
- session.connect();
- // exec 'scp -f rfile' remotely
- String command = "scp -f " + rfile;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- byte[] buf = new byte[1024];
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- while (true) {
- int c = checkAck(in);
- if (c != 'C') {
- break;
- }
-
- // read '0644 '
- in.read(buf, 0, 5);
-
- long filesize = 0L;
- while (true) {
- if (in.read(buf, 0, 1) < 0) {
- // error
- break;
- }
- if (buf[0] == ' ')
- break;
- filesize = filesize * 10L + (long) (buf[0] - '0');
- }
-
- String file = null;
- for (int i = 0;; i++) {
- in.read(buf, i, 1);
- if (buf[i] == (byte) 0x0a) {
- file = new String(buf, 0, i);
- break;
- }
- }
-
- //System.out.println("filesize="+filesize+", file="+file);
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- // read a content of lfile
- fos = new FileOutputStream(prefix == null ? lfile : prefix + file);
- int foo;
- while (true) {
- if (buf.length < filesize)
- foo = buf.length;
- else
- foo = (int) filesize;
- foo = in.read(buf, 0, foo);
- if (foo < 0) {
- // error
- break;
- }
- fos.write(buf, 0, foo);
- filesize -= foo;
- if (filesize == 0L)
- break;
- }
- fos.close();
- fos = null;
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- }
-
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fos != null)
- fos.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public SSHClientOutput execCommand(String command) throws Exception {
- return execCommand(command, 7200, null);
- }
-
- public SSHClientOutput execCommand(String command, int timeoutSeconds, Logger logAppender) throws Exception {
- try {
- System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
-
- StringBuffer text = new StringBuffer();
- int exitCode = -1;
-
- Session session = newJSchSession();
- session.connect();
-
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- channel.setInputStream(null);
-
- // channel.setOutputStream(System.out);
-
- ((ChannelExec) channel).setErrStream(System.err);
-
- InputStream in = channel.getInputStream();
- InputStream err = ((ChannelExec) channel).getErrStream();
-
- channel.connect();
-
- int timeout = timeoutSeconds;
- byte[] tmp = new byte[1024];
- while (true) {
- timeout--;
- while (in.available() > 0) {
- int i = in.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- while (err.available() > 0) {
- int i = err.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- if (channel.isClosed()) {
- if (in.available() > 0)
- continue;
- exitCode = channel.getExitStatus();
- System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
-
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (Exception ee) {
- throw ee;
- }
- if (timeout < 0)
- throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds.");
- }
- channel.disconnect();
- session.disconnect();
- return new SSHClientOutput(exitCode, text.toString());
- } catch (Exception e) {
- throw e;
- }
- }
-
- private Session newJSchSession() throws JSchException {
- JSch jsch = new JSch();
- if (identityPath != null) {
- jsch.addIdentity(identityPath);
- }
-
- Session session = jsch.getSession(username, hostname, 22);
- if (password != null) {
- session.setPassword(password);
- }
- session.setConfig("StrictHostKeyChecking", "no");
- return session;
- }
-
- private int checkAck(InputStream in) throws IOException {
- int b = in.read();
- // b may be 0 for success,
- // 1 for error,
- // 2 for fatal error,
- // -1
- if (b == 0)
- return b;
- if (b == -1)
- return b;
-
- if (b == 1 || b == 2) {
- StringBuffer sb = new StringBuffer();
- int c;
- do {
- c = in.read();
- sb.append((char) c);
- } while (c != '\n');
- if (b == 1) { // error
- System.out.print(sb.toString());
- }
- if (b == 2) { // fatal error
- System.out.print(sb.toString());
- }
- }
- return b;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java b/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
deleted file mode 100644
index a7e9e12..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class SSHClientOutput {
- private String text;
- private int exitCode = -1;
-
- /**
- * @param text
- * @param exitCode
- */
- public SSHClientOutput(int exitCode, String text) {
- this.text = text;
- this.exitCode = exitCode;
- }
-
- /**
- * @return the text
- */
- public String getText() {
- return text.toString();
- }
-
- /**
- * @return the exitCode
- */
- public int getExitCode() {
- return exitCode;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
deleted file mode 100644
index f82c8d6..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
-public class SoutLogger implements Logger {
-
- @Override
- public void log(String message) {
- System.out.println(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
deleted file mode 100644
index 8deeb5f..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class SplittedBytes {
- public SplittedBytes(int length) {
- this.value = new byte[length];
- this.length = 0;
- }
-
- public byte[] value;
- public int length;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java b/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
deleted file mode 100644
index 4b7ba7e..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class StringSplitter {
- public static String[] split(String str, String delimiter) {
- // The optimized split function
- List<String> list = new ArrayList<String>();
- int index = 0, offset = 0;
- int l = delimiter.length();
- if (str.startsWith(delimiter)) {
- // in case the first field is empty
- list.add("");
- offset = offset + l;
- }
- while ((index = str.indexOf(delimiter, index + 1)) != -1) {
- list.add(str.substring(offset, index));
- offset = index + l;
- }
- // add the last field, or the str doesn't contain delimiter at all
- list.add(str.substring(offset));
- return list.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
deleted file mode 100644
index dbf3380..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 12/2/13 Time: 11:43 AM To
- * change this template use File | Settings | File Templates.
- */
-public class StringUtil {
-
- public static String[] filterSystemArgs(String args[]) {
- ArrayList<String> whatsLeft = new ArrayList<String>();
- for (String a : args) {
- if (a.startsWith("-D")) {
- String key;
- String value;
- int cut = a.indexOf('=');
- if (cut < 0) {
- key = a.substring(2);
- value = "";
- } else {
- key = a.substring(2, cut);
- value = a.substring(cut + 1);
- }
- System.setProperty(key, value);
- } else {
- whatsLeft.add(a);
- }
- }
- return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
- }
-
- public static void toUpperCaseArray(String[] source, String[] target) {
- for (int i = 0; i < source.length; i++) {
- if (source[i] != null) {
- target[i] = source[i].toUpperCase();
- }
- }
- }
-
- public static String noBlank(String str, String dft) {
- return StringUtils.isBlank(str) ? dft : str;
- }
-
- public static String dropSuffix(String str, String suffix) {
- if (str.endsWith(suffix))
- return str.substring(0, str.length() - suffix.length());
- else
- return str;
- }
-
- public static String min(Collection<String> strs) {
- String min = null;
- for (String s : strs) {
- if (min == null || min.compareTo(s) > 0)
- min = s;
- }
- return min;
- }
-
- public static String max(Collection<String> strs) {
- String max = null;
- for (String s : strs) {
- if (max == null || max.compareTo(s) < 0)
- max = s;
- }
- return max;
- }
-
- public static String min(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) < 0 ? s1 : s2;
- }
-
- public static String max(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) > 0 ? s1 : s2;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java b/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
deleted file mode 100644
index 78751dc..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-public class TarGZUtil {
-
- public static void uncompressTarGZ(File tarFile, File dest) throws IOException {
- dest.mkdir();
- TarArchiveInputStream tarIn = null;
-
- tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile))));
-
- TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
- // tarIn is a TarArchiveInputStream
- while (tarEntry != null) {// create a file with the same name as the tarEntry
- File destPath = new File(dest, tarEntry.getName());
- System.out.println("working: " + destPath.getCanonicalPath());
- if (tarEntry.isDirectory()) {
- destPath.mkdirs();
- } else {
- destPath.createNewFile();
- //byte [] btoRead = new byte[(int)tarEntry.getSize()];
- byte[] btoRead = new byte[1024];
- //FileInputStream fin
- // = new FileInputStream(destPath.getCanonicalPath());
- BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath));
- int len = 0;
-
- while ((len = tarIn.read(btoRead)) != -1) {
- bout.write(btoRead, 0, len);
- }
-
- bout.close();
- btoRead = null;
-
- }
- tarEntry = tarIn.getNextTarEntry();
- }
- tarIn.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
deleted file mode 100644
index 37c939c..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/31/14.
- */
-public class ThreadUtil {
- @SuppressWarnings("unused")
- public static void main(String[] args) {
- ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable"));
-
- for (int i = 0; i < Integer.MAX_VALUE; ++i) {
- System.out.println("index: " + i);
- Future<?> future = pool.submit(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/resources/kylin-log4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/kylin-log4j.properties b/common/src/main/resources/kylin-log4j.properties
deleted file mode 100644
index 3772972..0000000
--- a/common/src/main/resources/kylin-log4j.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# enable this by -Dlog4j.configuration=kylin-log4j.properties
-
-log4j.rootLogger=INFO,stdout
-
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
-
-#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
-log4j.logger.org.springframework=WARN
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
deleted file mode 100644
index 75625fb..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
deleted file mode 100644
index 4a9daa6..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testFileStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
deleted file mode 100644
index 14538c6..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by honma on 9/18/14.
- */
-@Ignore("convenient trial tool for dev")
-public class ResourceToolTest {
- @Before
- public void setup() throws Exception {
- ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
- }
-
- @Test
- public void test() throws IOException {
- ResourceTool.copy(KylinConfig.createInstanceFromUri("../examples/test_case_data"), KylinConfig.createInstanceFromUri("../examples/test_case_data/kylin.properties"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java b/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
deleted file mode 100644
index f4d3410..0000000
--- a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-public class RestClientTest {
-
- @SuppressWarnings("unused")
- @Test
- public void basicTests() throws IOException {
- RestClient a = new RestClient("prod01:80");
- //a.wipeCache("metadata", "a", "a");
- //String aa = a.getKylinProperties();
- //System.out.println(aa);
- RestClient b = new RestClient("sandbox.hortonworks.com:7070");
- //b.wipeCache("metadata", "a", "a");
- //String bb = b.getKylinProperties();
- //System.out.println(bb);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
deleted file mode 100644
index 58d3ac1..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.lang.reflect.Method;
-
-import org.apache.kylin.common.KylinConfig;
-
-/**
- * @author ysong1
- *
- */
-public abstract class AbstractKylinTestCase {
-
- public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
-
- public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
- public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
-
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" };
-
- public abstract void createTestMetadata() throws Exception;
-
- public abstract void cleanupTestMetadata() throws Exception;
-
- public static KylinConfig getTestConfig() {
- return KylinConfig.getInstanceFromEnv();
- }
-
- public static void staticCreateTestMetadata(String kylinConfigFolder) {
-
- KylinConfig.destoryInstance();
-
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
-
- }
-
- public static void staticCleanupTestMetadata() {
- cleanupCache();
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
-
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
-
- }
-
- private static void cleanupCache() {
-
- for (String serviceClass : SERVICES_WITH_CACHE) {
- try {
- Class<?> cls = Class.forName(serviceClass);
- Method method = cls.getDeclaredMethod("clearCache");
- method.invoke(null);
- } catch (ClassNotFoundException e) {
- // acceptable because lower module test does have CubeManager etc on classpath
- } catch (Exception e) {
- System.err.println("Error clean up cache " + serviceClass);
- e.printStackTrace();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
deleted file mode 100644
index 6d2762c..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by honma on 11/11/14.
- *
- * development concept proving use
- */
-@Ignore("convenient trial tool for dev")
-public class BasicHadoopTest {
-
- @BeforeClass
- public static void setup() throws Exception {
- ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
- }
-
- @Test
- public void testCreateHtable() throws IOException {
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf("testhbase"));
- tableDesc.setValue("KYLIN_HOST", "dev01");
-
- HColumnDescriptor cf = new HColumnDescriptor("f");
- cf.setMaxVersions(1);
-
- cf.setInMemory(true);
- cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
- tableDesc.addFamily(cf);
-
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = new HBaseAdmin(conf);
- admin.createTable(tableDesc);
- admin.close();
- }
-
- @Test
- public void testRetriveHtableHost() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
- HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables();
- for (HTableDescriptor table : tableDescriptors) {
- String value = table.getValue("KYLIN_HOST");
- if (value != null) {
- System.out.println(table.getTableName());
- System.out.println("host is " + value);
- hbaseAdmin.disableTable(table.getTableName());
- table.setValue("KYLIN_HOST_ANOTHER", "dev02");
- hbaseAdmin.modifyTable(table.getTableName(), table);
- hbaseAdmin.enableTable(table.getTableName());
- }
- }
- hbaseAdmin.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
deleted file mode 100644
index f2778d2..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
-
-/**
-* Created by honma on 10/17/14.
-* <p/>
-* Keep this test case to test basic java functionality
-* development concept proving use
-*/
-@Ignore("convenient trial tool for dev")
-@SuppressWarnings("unused")
-public class BasicTest {
- protected static final org.slf4j.Logger log = LoggerFactory.getLogger(BasicTest.class);
-
- private void log(ByteBuffer a) {
- Integer x = 4;
- foo(x);
- }
-
- private void foo(Long a) {
- System.out.printf("a");
-
- }
-
- private void foo(Integer b) {
- System.out.printf("b");
- }
-
- private enum MetricType {
- Count, DimensionAsMetric, DistinctCount, Normal
- }
-
- @Test
- @Ignore("convenient trial tool for dev")
- public void test1() throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(10);
- ListenableFuture futureTask = MoreExecutors.listeningDecorator(executorService).submit(new Runnable() {
- @Override
- public void run() {
-
- }
- });
- futureTask.addListener(new Runnable() {
- @Override
- public void run() {
-
- }
- }, executorService);
-
- }
-
- @Test
- @Ignore("fix it later")
- public void test2() throws IOException, ConfigurationException {
- int m = 1 << 15;
- System.out.println(m);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
deleted file mode 100644
index 178e568..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.nio.ByteBuffer;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-
-/**
- * by honma
- */
-public class BytesUtilTest extends TestCase {
- @Test
- public void test() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- int[] x = new int[] { 1, 2, 3 };
- BytesUtil.writeIntArray(x, buffer);
- buffer.flip();
-
- byte[] buf = new byte[buffer.limit()];
- System.arraycopy(buffer.array(), 0, buf, 0, buffer.limit());
-
- ByteBuffer newBuffer = ByteBuffer.wrap(buf);
- int[] y = BytesUtil.readIntArray(newBuffer);
- assertEquals(y[2], 3);
- }
-
- @Test
- public void testBooleanArray() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- boolean[] x = new boolean[] { true, false, true };
- BytesUtil.writeBooleanArray(x, buffer);
- buffer.flip();
- boolean[] y = BytesUtil.readBooleanArray(buffer);
- assertEquals(y[2], true);
- assertEquals(y[1], false);
- }
-
- @Test
- public void testReadable() {
- String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
- byte[] bytes = BytesUtil.fromReadableText(x);
- String y = BytesUtil.toHex(bytes);
- assertEquals(x, y);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
deleted file mode 100644
index d2e3238..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * @author ysong1
- */
-public class HBaseMetadataTestCase extends AbstractKylinTestCase {
-
- static {
- if (useSandbox()) {
- try {
- File sandboxFolder = new File("../examples/test_case_data/sandbox/");
- if (sandboxFolder.exists() == false) {
- throw new IOException("The sandbox folder doesn't exist: " + sandboxFolder.getAbsolutePath());
- }
- ClassUtil.addClasspath(sandboxFolder.getAbsolutePath());
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
-
- @Override
- public void createTestMetadata() throws Exception {
- staticCreateTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- staticCleanupTestMetadata();
- }
-
- public static void staticCreateTestMetadata() throws Exception {
- if (useSandbox()) {
- staticCreateTestMetadata(SANDBOX_TEST_DATA);
- } else {
- staticCreateTestMetadata(MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- }
-
- }
-
- public static boolean useSandbox() {
- String useSandbox = System.getProperty("useSandbox");
- if (StringUtils.isEmpty(useSandbox)) {
- return true;
- }
-
- return Boolean.parseBoolean(useSandbox);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
deleted file mode 100644
index 6f31776..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseResourceStore;
-
-/**
- * a helper class to start and shutdown hbase mini cluster
- *
- * @author shaoshi
- */
-public class HBaseMiniclusterHelper {
-
- public static final String SHARED_STORAGE_PREFIX = "KYLIN_";
- public static final String CUBE_STORAGE_PREFIX = "KYLIN_";
- public static final String II_STORAGE_PREFIX = "KYLIN_II";
- public static final String TEST_METADATA_TABLE = "kylin_metadata";
-
- private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
- private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
-
- public static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static volatile boolean clusterStarted = false;
- private static String zkHost, zkPort, zkParent;
-
- private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class);
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- shutdownMiniCluster();
- }
- });
- }
-
- /**
- * Start the minicluster; Sub-classes should invoke this in BeforeClass method.
- *
- * @throws Exception
- */
- public static void startupMinicluster() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA);
-
- if (!clusterStarted) {
- synchronized (HBaseMiniclusterHelper.class) {
- if (!clusterStarted) {
- startupMiniClusterAndImportData();
- clusterStarted = true;
- }
- }
- } else {
- updateKylinConfigWithMinicluster();
- }
- }
-
- private static void updateKylinConfigWithMinicluster() {
-
- Configuration hbaseConfiguration = HadoopUtil.getCurrentHBaseConfiguration();
- hbaseConfiguration.set(HConstants.ZOOKEEPER_QUORUM, zkHost);
- hbaseConfiguration.set(HConstants.ZOOKEEPER_CLIENT_PORT, zkPort);
- hbaseConfiguration.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent);
- }
-
- private static void startupMiniClusterAndImportData() throws Exception {
-
- logger.info("Going to start mini cluster.");
-
- if (existInClassPath(iiEndpointClassName)) {
- HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName);
- }
-
- //https://issues.apache.org/jira/browse/HBASE-11711
- UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering
-
- MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster();
- UTIL.startMiniMapReduceCluster();
- Configuration config = hbaseCluster.getConf();
- zkHost = config.get(HConstants.ZOOKEEPER_QUORUM);
- zkPort = config.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- zkParent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
-
- // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired
- config.set("zookeeper.session.timeout", "1200000");
- config.set("hbase.zookeeper.property.tickTime", "6000");
- // reduce rpc retry
- config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
- config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- updateKylinConfigWithMinicluster();
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- // create the metadata htables;
- @SuppressWarnings("unused")
- HBaseResourceStore store = new HBaseResourceStore(kylinConfig);
-
- // import the table content
- HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration());
-
- }
-
- private static boolean existInClassPath(String className) {
- try {
- Class.forName(className);
- } catch (ClassNotFoundException e) {
- return false;
- }
- return true;
- }
-
- /**
- * Shutdown the minicluster;
- */
- public static void shutdownMiniCluster() {
-
- logger.info("Going to shutdown mini cluster.");
-
- try {
- UTIL.shutdownMiniMapReduceCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- HBaseMiniclusterHelper t = new HBaseMiniclusterHelper();
- logger.info(t);
- try {
- HBaseMiniclusterHelper.startupMinicluster();
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- HBaseMiniclusterHelper.shutdownMiniCluster();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
deleted file mode 100644
index 8587683..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-
-import static org.junit.Assert.*;
-
-/**
- * Created by sunyerui on 15/8/26.
- * Tests for HadoopUtil
- */
-public class HadoopUtilTest {
-
- @BeforeClass
- public static void beforeClass() {
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- }
-
- @After
- public void after() {
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
- }
-
- @Test
- public void testGetCurrentHBaseConfiguration() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "hdfs://hbase-cluster/");
-
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- assertEquals("hdfs://hbase-cluster/", conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
- }
-
- @Test
- public void testMakeQualifiedPathInHBaseCluster() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "file:/");
-
- String path = HadoopUtil.makeQualifiedPathInHBaseCluster("/path/to/test/hbase");
- assertEquals("file:/path/to/test/hbase", path);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
deleted file mode 100644
index 5d8d6d6..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.mapreduce.Import;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-
-import com.google.common.base.Preconditions;
-
-public class HbaseImporter {
-
- private static final Log logger = LogFactory.getLog(HbaseImporter.class);
-
- public static void importHBaseData(String hbaseTarLocation, Configuration conf) throws IOException, ClassNotFoundException, InterruptedException {
-
- if (System.getenv("JAVA_HOME") == null) {
- logger.error("Didn't find $JAVA_HOME, this will cause HBase data import failed. Please set $JAVA_HOME.");
- logger.error("Skipping table import...");
- return;
- }
-
- File exportFile = new File(hbaseTarLocation);
- if (!exportFile.exists()) {
- logger.error("Didn't find the export achieve file on " + exportFile.getAbsolutePath());
- return;
- }
-
- File folder = File.createTempFile("hbase-import", "tmp");
- if (folder.exists()) {
- FileUtils.forceDelete(folder);
- }
- folder.mkdirs();
- FileUtils.forceDeleteOnExit(folder);
-
- //TarGZUtil.uncompressTarGZ(exportFile, folder);
- FileUtil.unTar(exportFile, folder);
- String[] child = folder.list();
- Preconditions.checkState(child.length == 1);
- String backupFolderName = child[0];
- File backupFolder = new File(folder, backupFolderName);
- String[] tableNames = backupFolder.list();
-
- for (String table : tableNames) {
-
- if (!(table.equalsIgnoreCase(HBaseMiniclusterHelper.TEST_METADATA_TABLE) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) {
- continue;
- }
-
- // create the htable; otherwise the import will fail.
- if (table.startsWith(HBaseMiniclusterHelper.II_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "f");
- } else if (table.startsWith(HBaseMiniclusterHelper.CUBE_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2");
- }
-
- // directly import from local fs, no need to copy to hdfs
- String importLocation = "file://" + backupFolder.getAbsolutePath() + "/" + table;
- String[] args = new String[] { table, importLocation };
- boolean result = runImport(args, conf);
- logger.info("importing table '" + table + "' with result:" + result);
-
- if (!result)
- break;
- }
-
- }
-
- private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException {
- // need to make a copy of the configuration because to make sure different temp dirs are used.
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args);
- Configuration newConf = opts.getConfiguration();
- args = opts.getRemainingArgs();
- Job job = Import.createSubmittableJob(newConf, args);
- job.waitForCompletion(false);
- return job.isSuccessful();
- }
-
- public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
- if (args.length != 1) {
- logger.error("Usage: HbaseImporter hbase_tar_lcoation");
- System.exit(-1);
- }
-
- logger.info("The KylinConfig being used:");
- logger.info("=================================================");
- KylinConfig.getInstanceFromEnv().printProperties();
- logger.info("=================================================");
-
- importHBaseData(args[0], HBaseConfiguration.create());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
deleted file mode 100644
index ff42713..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class HyperLogLogCounterTest {
-
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- Random rand1 = new Random(1);
- Random rand2 = new Random(2);
- Random rand3 = new Random(3);
- int errorCount1 = 0;
- int errorCount2 = 0;
- int errorCount3 = 0;
-
- private Set<String> generateTestData(int n) {
- Set<String> testData = new HashSet<String>();
- for (int i = 0; i < n; i++) {
- String[] samples = generateSampleData();
- for (String sample : samples) {
- testData.add(sample);
- }
- }
- return testData;
- }
-
- // simulate the visit (=visitor+id)
- private String[] generateSampleData() {
-
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < 19; i++) {
- buf.append(Math.abs(rand1.nextInt()) % 10);
- }
- String header = buf.toString();
-
- int size = Math.abs(rand3.nextInt()) % 9 + 1;
- String[] samples = new String[size];
- for (int k = 0; k < size; k++) {
- buf = new StringBuilder(header);
- buf.append("-");
- for (int i = 0; i < 10; i++) {
- buf.append(Math.abs(rand3.nextInt()) % 10);
- }
- samples[k] = buf.toString();
- }
-
- return samples;
- }
-
- @Test
- public void countTest() throws IOException {
- int n = 10;
- for (int i = 0; i < 5; i++) {
- count(n);
- n *= 10;
- }
- }
-
- private void count(int n) throws IOException {
- Set<String> testSet = generateTestData(n);
-
- HyperLogLogPlusCounter hllc = newHLLC();
- for (String testData : testSet) {
- hllc.add(Bytes.toBytes(testData));
- }
- long estimate = hllc.getCountEstimate();
- double errorRate = hllc.getErrorRate();
- double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
- System.out.println(estimate);
- System.out.println(testSet.size());
- System.out.println(errorRate);
- System.out.println("=" + actualError);
- Assert.assertTrue(actualError < errorRate * 3.0);
-
- checkSerialize(hllc);
- }
-
- private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
- long estimate = hllc.getCountEstimate();
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- hllc.readRegisters(buf);
- Assert.assertEquals(estimate, hllc.getCountEstimate());
- }
-
- @Test
- public void mergeTest() throws IOException {
- double error = 0;
- double absError = 0;
- int n = 100;
- for (int i = 0; i < n; i++) {
- double e = merge();
- error += e;
- absError += Math.abs(e);
- }
- System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n);
-
- System.out.println(" errorRateCount1 is " + errorCount1 + "!");
- System.out.println(" errorRateCount2 is " + errorCount2 + "!");
- System.out.println(" errorRateCount3 is " + errorCount3 + "!");
-
- Assert.assertTrue(errorCount1 <= n * 0.40);
- Assert.assertTrue(errorCount2 <= n * 0.08);
- Assert.assertTrue(errorCount3 <= n * 0.02);
- }
-
- private double merge() throws IOException {
-
- int ln = 50;
- int dn = 300;
- Set<String> testSet = new HashSet<String>();
- HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln];
- for (int i = 0; i < ln; i++) {
- hllcs[i] = newHLLC();
- for (int k = 0; k < dn; k++) {
- String[] samples = generateSampleData();
- for (String data : samples) {
- testSet.add(data);
- hllcs[i].add(Bytes.toBytes(data));
- }
- }
- }
- HyperLogLogPlusCounter mergeHllc = newHLLC();
- for (HyperLogLogPlusCounter hllc : hllcs) {
- mergeHllc.merge(hllc);
- checkSerialize(mergeHllc);
- }
-
- double errorRate = mergeHllc.getErrorRate();
- long estimate = mergeHllc.getCountEstimate();
- double actualError = (double) (testSet.size() - estimate) / testSet.size();
-
- System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
-
- if (Math.abs(actualError) > errorRate) {
- errorCount1++;
- }
- if (Math.abs(actualError) > 2 * errorRate) {
- errorCount2++;
- }
- if (Math.abs(actualError) > 3 * errorRate) {
- errorCount3++;
- }
-
- return actualError;
- }
-
- @Test
- public void testPerformance() throws IOException {
- int N = 3; // reduce N HLLC into one
- int M = 1000; // for M times, use 100000 for real perf test
-
- HyperLogLogPlusCounter samples[] = new HyperLogLogPlusCounter[N];
- for (int i = 0; i < N; i++) {
- samples[i] = newHLLC();
- for (String str : generateTestData(10000))
- samples[i].add(str);
- }
-
- System.out.println("Perf test running ... ");
- long start = System.currentTimeMillis();
- HyperLogLogPlusCounter sum = newHLLC();
- for (int i = 0; i < M; i++) {
- sum.clear();
- for (int j = 0; j < N; j++) {
- sum.merge(samples[j]);
- checkSerialize(sum);
- }
- }
- long duration = System.currentTimeMillis() - start;
- System.out.println("Perf test result: " + duration / 1000 + " seconds");
- }
-
- @Test
- public void testEquivalence() {
- byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
- byte[] b = new byte[] { 3, 4, 42 };
- HyperLogLogPlusCounter ha = new HyperLogLogPlusCounter();
- HyperLogLogPlusCounter hb = new HyperLogLogPlusCounter();
- ha.add(a, 1, 3);
- hb.add(b);
-
- Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
- }
-
- private HyperLogLogPlusCounter newHLLC() {
- return new HyperLogLogPlusCounter(16);
- }
-}
[17/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
deleted file mode 100644
index 86b0543..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author honma
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IIInstance extends RootPersistentEntity implements IRealization {
-
- public static IIInstance create(String iiName, String projectName, IIDesc iiDesc) {
- IIInstance iii = new IIInstance();
-
- iii.setConfig(iiDesc.getConfig());
- iii.setName(iiName);
- iii.setDescName(iiDesc.getName());
- iii.setCreateTimeUTC(System.currentTimeMillis());
- iii.setStatus(RealizationStatusEnum.DISABLED);
- iii.updateRandomUuid();
-
- return iii;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private RealizationStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<IISegment> segments = new ArrayList<IISegment>();
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- private String projectName;
-
- public long getAllocatedEndDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(segments.size() - 1).getDateRangeEnd();
- }
-
- public long getAllocatedStartDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(0).getDateRangeStart();
- }
-
- public IIDesc getDescriptor() {
- return IIDescManager.getInstance(config).getIIDesc(descName);
- }
-
- public boolean isReady() {
- return getStatus() == RealizationStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.II_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return getCanonicalName();
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("input_records_count")
- public long getInputRecordCount() {
- long sizeRecordCount = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getInputRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("input_records_size")
- public long getInputRecordSize() {
- long sizeRecordSize = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- @Override
- public String getCanonicalName() {
- return getType() + "[name=" + name + "]";
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(RealizationStatusEnum status) {
- this.status = status;
- }
-
- public IISegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public List<IISegment> getSegments() {
- return segments;
- }
-
- public List<IISegment> getSegments(SegmentStatusEnum status) {
- List<IISegment> result = new ArrayList<IISegment>();
-
- for (IISegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public IISegment getSegment(String name, SegmentStatusEnum status) {
- for (IISegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<IISegment> segments) {
- this.segments = segments;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- @Override
- public CapabilityResult isCapable(SQLDigest digest) {
- CapabilityResult result = IICapabilityChecker.check(this, digest);
- if (result.capable) {
- result.cost = getCost(digest);
- } else {
- result.cost = -1;
- }
- return result;
- }
-
- public int getCost(SQLDigest digest) {
- return 0;
- }
-
- @Override
- public RealizationType getType() {
- return RealizationType.INVERTED_INDEX;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return getDescriptor().listAllColumns();
- }
-
- @Override
- public String getFactTable() {
- return getDescriptor().getFactTableName();
- }
-
- @Override
- public List<MeasureDesc> getMeasures() {
- return getDescriptor().getMeasures();
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
- @Override
- public long getDateRangeStart() {
- List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (IISegment seg : readySegs) {
- if (seg.getDateRangeStart() < startTime)
- startTime = seg.getDateRangeStart();
- }
-
- return startTime;
- }
-
- @Override
- public long getDateRangeEnd() {
-
- List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = 0;
- for (IISegment seg : readySegs) {
- if (seg.getDateRangeEnd() > endTime)
- endTime = seg.getDateRangeEnd();
- }
-
- return endTime;
- }
-
- @Override
- public List<TblColRef> getAllDimensions() {
- return getDescriptor().listAllDimensions();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
deleted file mode 100644
index f7e70f4..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.IRealizationProvider;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-public class IIManager implements IRealizationProvider {
-
- private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
- private static int HBASE_TABLE_LENGTH = 10;
-
- private static final Serializer<IIInstance> II_SERIALIZER = new JsonSerializer<IIInstance>(IIInstance.class);
-
- private static final Logger logger = LoggerFactory.getLogger(IIManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, IIManager> CACHE = new ConcurrentHashMap<KylinConfig, IIManager>();
-
- public static IIManager getInstance(KylinConfig config) {
- IIManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (IIManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new IIManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init IIManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // ii name ==> IIInstance
- private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX);
-
- // for generation hbase table name of a new segment
- private HashSet<String> usedStorageLocation = new HashSet<String>();
-
- private IIManager(KylinConfig config) throws IOException {
- logger.info("Initializing IIManager with config " + config);
- this.config = config;
-
- loadAllIIInstance();
- }
-
- public List<IIInstance> listAllIIs() {
- return new ArrayList<IIInstance>(iiMap.values());
- }
-
- public IIInstance getII(String iiName) {
- iiName = iiName.toUpperCase();
- return iiMap.get(iiName);
- }
-
- public List<IIInstance> getIIsByDesc(String descName) {
-
- List<IIInstance> list = listAllIIs();
- List<IIInstance> result = new ArrayList<IIInstance>();
- Iterator<IIInstance> it = list.iterator();
- while (it.hasNext()) {
- IIInstance ci = it.next();
- if (descName.equalsIgnoreCase(ci.getDescName())) {
- result.add(ci);
- }
- }
- return result;
- }
-
- public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException {
- logger.info("Start building ii dictionary");
- DictionaryManager dictMgr = getDictionaryManager();
- IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor();
- for (TblColRef column : iiDesc.listAllColumns()) {
- logger.info("Dealing with column {}", column);
- if (iiDesc.isMetricsCol(column)) {
- continue;
- }
-
- DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
- iiSeg.putDictResPath(column, dict.getResourcePath());
- }
- saveResource(iiSeg.getIIInstance());
- }
-
- /**
- * return null if no dictionary for given column
- */
- public Dictionary<?> getDictionary(IISegment iiSeg, TblColRef col) {
- DictionaryInfo info = null;
- try {
- DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
- String dictResPath = iiSeg.getDictResPath(col);
- if (dictResPath == null)
- return null;
-
- info = dictMgr.getDictionaryInfo(dictResPath);
- if (info == null)
- throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid II state; II segment" + iiSeg + ", col " + col);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to get dictionary for II segment" + iiSeg + ", col" + col, e);
- }
-
- return info.getDictionaryObject();
- }
-
- public IIInstance createII(IIInstance ii) throws IOException {
-
- if (this.getII(ii.getName()) != null)
- throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
-
- // other logic is the same as update.
- return updateII(ii);
- }
-
- public IIInstance updateII(IIInstance ii) throws IOException {
- logger.info("Updating II instance '" + ii.getName());
-
- // save resource
- saveResource(ii);
-
- logger.info("II with " + ii.getSegments().size() + " segments is saved");
-
- return ii;
- }
-
- public void loadIICache(String iiName) {
- try {
- loadIIInstance(IIInstance.concatResourcePath(iiName));
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
-
- public void removeIICache(IIInstance ii) {
- iiMap.remove(ii.getName());
-
- for (IISegment segment : ii.getSegments()) {
- usedStorageLocation.remove(segment.getName());
- }
- }
-
- public void removeIILocalCache(String name) {
- iiMap.removeLocal(name);
- //TODO
- // for (IISegment segment : ii.getSegments()) {
- // usedStorageLocation.remove(segment.getName());
- // }
- }
-
- private void saveResource(IIInstance ii) throws IOException {
- ResourceStore store = getStore();
- store.putResource(ii.getResourcePath(), ii, II_SERIALIZER);
- this.afterIIUpdated(ii);
- }
-
- private void afterIIUpdated(IIInstance updatedII) {
- iiMap.put(updatedII.getName(), updatedII);
- }
-
- /**
- * @param IIInstance
- * @param startDate (pass 0 if full build)
- * @param endDate (pass 0 if full build)
- * @return
- */
- public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) {
- IISegment segment = new IISegment();
- String incrementalSegName = IISegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- segment.setCreateTimeUTC(System.currentTimeMillis());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setIIInstance(IIInstance);
-
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.IIHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- int idx = (int) (Math.random() * ALPHA_NUM.length());
- sb.append(ALPHA_NUM.charAt(idx));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.contains(tableName));
-
- return tableName;
- }
-
- private void loadAllIIInstance() throws IOException {
- ResourceStore store = getStore();
- List<String> paths = store.collectResourceRecursively(ResourceStore.II_RESOURCE_ROOT, ".json");
-
- logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));
-
- for (String path : paths) {
- loadIIInstance(path);
- }
-
- logger.debug("Loaded " + paths.size() + " II(s)");
- }
-
- private synchronized IIInstance loadIIInstance(String path) throws IOException {
- ResourceStore store = getStore();
- logger.debug("Loading IIInstance " + store.getReadableResourcePath(path));
-
- IIInstance IIInstance = null;
- try {
- IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER);
- IIInstance.setConfig(config);
-
- if (StringUtils.isBlank(IIInstance.getName()))
- throw new IllegalStateException("IIInstance name must not be blank");
-
- iiMap.putLocal(IIInstance.getName(), IIInstance);
-
- for (IISegment segment : IIInstance.getSegments()) {
- usedStorageLocation.add(segment.getName());
- }
-
- return IIInstance;
- } catch (Exception e) {
- logger.error("Error during load ii instance " + path, e);
- return null;
- }
- }
-
- private DictionaryManager getDictionaryManager() {
- return DictionaryManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- @Override
- public RealizationType getRealizationType() {
- return RealizationType.INVERTED_INDEX;
- }
-
- @Override
- public IRealization getRealization(String name) {
- return getII(name);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
deleted file mode 100644
index 77a876c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-/**
- * @author honma
- */
-
-// TODO: remove segment concept for II, append old hbase table
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IISegment implements Comparable<IISegment>, ISegment {
-
- @JsonBackReference
- private IIInstance iiInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private SegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("input_records")
- private long inputRecords;
- @JsonProperty("input_records_size")
- private long inputRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of schema and dictionary ID,
- // used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==>
- // dictionary
- // resource path
-
- private transient TableRecordInfo tableRecordInfo;
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- public IIDesc getIIDesc() {
- return getIIInstance().getDescriptor();
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public SegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(SegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getInputRecords() {
- return inputRecords;
- }
-
- public void setInputRecords(long inputRecords) {
- this.inputRecords = inputRecords;
- }
-
- public long getInputRecordsSize() {
- return inputRecordsSize;
- }
-
- public void setInputRecordsSize(long inputRecordsSize) {
- this.inputRecordsSize = inputRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public IIInstance getIIInstance() {
- return iiInstance;
- }
-
- public void setIIInstance(IIInstance iiInstance) {
- this.iiInstance = iiInstance;
- }
-
- public String getStorageLocationIdentifier() {
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- /**
- * @param storageLocationIdentifier the storageLocationIdentifier to set
- */
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int compareTo(IISegment other) {
- if (this.dateRangeEnd < other.dateRangeEnd) {
- return -1;
- } else if (this.dateRangeEnd > other.dateRangeEnd) {
- return 1;
- } else {
- return 0;
- }
- }
-
- private TableRecordInfo getTableRecordInfo() {
- if (tableRecordInfo == null)
- tableRecordInfo = new TableRecordInfo(this);
- return tableRecordInfo;
- }
-
- public List<TblColRef> getColumns() {
- return this.getTableRecordInfo().getColumns();
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
-
- int index = getTableRecordInfo().findColumn(col);
- return getTableRecordInfo().getDigest().length(index);
- }
-
- @Override
- public Dictionary<String> getDictionary(TblColRef col) {
-
- int index = getTableRecordInfo().findColumn(col);
- return getTableRecordInfo().dict(index);
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
deleted file mode 100644
index 164e2b9..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public class BitMapContainer implements ColumnValueContainer {
-
- int valueLen;
- int nValues;
- int size;
- RoaringBitmap[] sets;
- boolean closedForChange;
-
- transient byte[] temp;
-
- public BitMapContainer(TableRecordInfoDigest digest, int col) {
- this.valueLen = digest.length(col);
- this.size = 0;
- this.nValues = digest.getMaxID(col) + 1;
- this.sets = null;
- this.closedForChange = false;
-
- this.temp = new byte[valueLen];
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
- append(value);
- }
-
- public void append(int value) {
- checkUpdateMode();
- if (value == Dictionary.NULL_ID[valueLen]) {
- value = nValues; // set[nValues] holds NULL
- }
- sets[value].add(size);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- int value = getValueIntAt(i);
- BytesUtil.writeUnsigned(value, temp, 0, valueLen);
- valueBytes.set(temp, 0, valueLen);
- }
-
- @Override
- public RoaringBitmap getBitMap(Integer startId, Integer endId) {
- if (startId == null && endId == null) {
- return sets[this.nValues];
- }
-
- int start = 0;
- int end = this.nValues - 1;
- if (startId != null) {
- start = startId;
- }
- if (endId != null) {
- end = endId;
- }
-
- return RoaringBitmap.or(Arrays.copyOfRange(sets, start, end + 1));
- }
-
- @SuppressWarnings("unused")
- private RoaringBitmap getBitMap(int valueId) {
- if (valueId >= 0 && valueId <= getMaxValueId())
- return sets[valueId];
- else
- return sets[this.nValues];
- }
-
- @Override
- public int getMaxValueId() {
- return this.nValues - 1;
- }
-
- public int getValueIntAt(int i) {
- for (int v = 0; v < nValues; v++) {
- if (sets[v].contains(i)) {
- return v;
- }
- }
- // if v is not in [0..nValues-1], then it must be nValue (NULL)
- return Dictionary.NULL_ID[valueLen];
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalStateException();
- }
- if (sets == null) {
- sets = new RoaringBitmap[nValues + 1];
- for (int i = 0; i <= nValues; i++) {
- sets[i] = new RoaringBitmap();
- }
- }
- }
-
- private boolean isClosedForChange() {
- return closedForChange;
- }
-
- @Override
- public void closeForChange() {
- closedForChange = true;
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public List<ImmutableBytesWritable> toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
-
- List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
- for (int i = 0; i <= nValues; i++) {
- r.add(setToBytes(sets[i]));
- }
- return r;
- }
-
- public void fromBytes(List<ImmutableBytesWritable> bytes) {
- assert nValues + 1 == bytes.size();
- sets = new RoaringBitmap[nValues + 1];
- size = 0;
- for (int i = 0; i <= nValues; i++) {
- sets[i] = bytesToSet(bytes.get(i));
- size += sets[i].getCardinality();
- }
- closedForChange = true;
- }
-
- private ImmutableBytesWritable setToBytes(RoaringBitmap set) {
- // Serializing a bitmap to a byte array can be expected to be expensive, this should not be commonly done.
- // If the purpose is to save the data to disk or to a network, then a direct serialization would be
- // far more efficient. If the purpose is to enforce immutability, it is an expensive way to do it.
- set.runOptimize(); //to improve compression
- final byte[] array = new byte[set.serializedSizeInBytes()];
- try {
- set.serialize(new java.io.DataOutputStream(new java.io.OutputStream() {
- int c = 0;
-
- @Override
- public void close() {
- }
-
- @Override
- public void flush() {
- }
-
- @Override
- public void write(int b) {
- array[c++] = (byte) b;
- }
-
- @Override
- public void write(byte[] b) {
- write(b, 0, b.length);
- }
-
- @Override
- public void write(byte[] b, int off, int l) {
- System.arraycopy(b, off, array, c, l);
- c += l;
- }
- }));
- } catch (IOException ioe) {
- // should never happen because we write to a byte array
- throw new RuntimeException("unexpected error while serializing to a byte array");
- }
-
- return new ImmutableBytesWritable(array);
- }
-
- private RoaringBitmap bytesToSet(final ImmutableBytesWritable bytes) {
- // converting a byte array to a bitmap can be expected to be expensive, hopefully this is not a common operation!
- RoaringBitmap set = new RoaringBitmap();
- if ((bytes.get() != null) && (bytes.getLength() > 0)) {
- // here we could use an ImmutableRoaringBitmap and just "map" it.
- // instead, we do a full deserialization
- // Note: we deserializing a Roaring bitmap, there is no need to know the length, the format is self-describing
- try {
- set.deserialize(new java.io.DataInputStream(new java.io.InputStream() {
- byte[] array = bytes.get();
- int c = bytes.getOffset();
-
- @Override
- public int read() {
- return array[c++] & 0xff;
- }
-
- @Override
- public int read(byte b[]) {
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(byte[] b, int off, int l) {
- System.arraycopy(array, c, b, off, l);
- c += l;
- return l;
- }
- }));
- } catch (IOException ioe) {
- // should never happen because we read from a byte array
- throw new RuntimeException("unexpected error while deserializing from a byte array");
- }
- }
- return set;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (closedForChange ? 1231 : 1237);
- result = prime * result + nValues;
- result = prime * result + Arrays.hashCode(sets);
- result = prime * result + size;
- result = prime * result + valueLen;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BitMapContainer other = (BitMapContainer) obj;
- if (closedForChange != other.closedForChange)
- return false;
- if (nValues != other.nValues)
- return false;
- if (!Arrays.equals(sets, other.sets))
- return false;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
deleted file mode 100644
index a5ce1bd..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public interface ColumnValueContainer {
-
- void append(ImmutableBytesWritable valueBytes);
-
- void closeForChange();
-
- int getSize();
-
- // works only after closeForChange()
- void getValueAt(int i, ImmutableBytesWritable valueBytes);
-
- RoaringBitmap getBitMap(Integer startId, Integer endId);
-
- int getMaxValueId();
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
deleted file mode 100644
index 334457c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public class CompressedValueContainer implements ColumnValueContainer {
- int valueLen;
- int nValues;
- int cap;
- int size;
- byte[] uncompressed;
- byte[] compressed;
-
- public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
- this.valueLen = digest.length(col);
- this.nValues = digest.getMaxID(col) + 1;
- this.cap = cap;
- this.size = 0;
- this.uncompressed = null;
- this.compressed = null;
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- checkUpdateMode();
- System.arraycopy(valueBytes.get(), valueBytes.getOffset(), uncompressed, valueLen * size, valueLen);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- valueBytes.set(uncompressed, valueLen * i, valueLen);
- }
-
- @Override
- public RoaringBitmap getBitMap(Integer startId, Integer endId) {
- RoaringBitmap ret = new RoaringBitmap();
- int nullId = Dictionary.NULL_ID[valueLen];
-
- if (startId == null && endId == null) {
- //entry for getting null values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (nullId == valueID) {
- ret.add(i);
- }
- }
- return ret;
- }
-
- //normal values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (valueID == nullId) {
- continue;
- }
-
- if (startId != null && valueID < startId) {
- continue;
- }
-
- if (endId != null && valueID > endId) {
- continue;
- }
-
- ret.add(i);
- }
- return ret;
-
- }
-
- @Override
- public int getMaxValueId() {
- return nValues - 1;
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalArgumentException();
- }
- if (uncompressed == null) {
- uncompressed = new byte[valueLen * cap];
- }
- }
-
- private boolean isClosedForChange() {
- return compressed != null;
- }
-
- @Override
- public void closeForChange() {
- checkUpdateMode();
- try {
- compressed = LZFEncoder.encode(uncompressed, 0, valueLen * size);
- } catch (Exception e) {
- throw new RuntimeException("LZF encode failure", e);
- }
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public ImmutableBytesWritable toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
- return new ImmutableBytesWritable(compressed);
- }
-
- public void fromBytes(ImmutableBytesWritable bytes) {
- try {
- uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- size = cap = uncompressed.length / valueLen;
- compressed = BytesUtil.EMPTY_BYTE_ARRAY; // mark closed
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + size;
- result = prime * result + valueLen;
- result = prime * result + Arrays.hashCode(uncompressed);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CompressedValueContainer other = (CompressedValueContainer) obj;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- if (!Bytes.equals(uncompressed, 0, size * valueLen, uncompressed, 0, size * valueLen))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
deleted file mode 100644
index b9f963e..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-
-/**
- * Created by honma on 11/10/14.
- */
-public class RawTableRecord implements Cloneable {
- TableRecordInfoDigest digest;
- private byte[] buf; // consecutive column value IDs (encoded by dictionary)
-
- public RawTableRecord(TableRecordInfoDigest info) {
- this.digest = info;
- this.buf = new byte[info.getByteFormLen()];
- reset();
- }
-
- public RawTableRecord(RawTableRecord another) {
- this.digest = another.digest;
- this.buf = Bytes.copy(another.buf);
- }
-
- public void reset() {
- Arrays.fill(buf, Dictionary.NULL);
- }
-
- public boolean isMetric(int col) {
- return digest.isMetrics(col);
- }
-
- public FixedLenMeasureCodec<LongWritable> codec(int col) {
- return digest.codec(col);
- }
-
- public int length(int col) {
- return digest.length(col);
- }
-
- public int getColumnCount() {
- return digest.getColumnCount();
- }
-
- public void setValueID(int col, int id) {
- BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col));
- }
-
- public int getValueID(int col) {
- return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col));
- }
-
- public void setValueMetrics(int col, LongWritable value) {
- digest.codec(col).write(value, buf, digest.offset(col));
- }
-
- public String getValueMetric(int col) {
- digest.codec(col).read(buf, digest.offset(col));
- return (String) digest.codec(col).getValue();
- }
-
- public byte[] getBytes() {
- return buf;
- }
-
- //TODO is it possible to avoid copying?
- public void setBytes(byte[] bytes, int offset, int length) {
- assert buf.length == length;
- System.arraycopy(bytes, offset, buf, 0, length);
- }
-
- public void setValueBytes(int col, ImmutableBytesWritable bytes) {
- System.arraycopy(bytes.get(), bytes.getOffset(), buf, digest.offset(col), digest.length(col));
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- bytes.set(buf, digest.offset(col), digest.length(col));
- }
-
- @Override
- public Object clone() {
- return new RawTableRecord(this);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(buf);
- // result = prime * result + ((digest == null) ? 0 : digest.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RawTableRecord other = (RawTableRecord) obj;
- if (!Arrays.equals(buf, other.buf))
- return false;
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
deleted file mode 100644
index 0fd1bf9..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-public class ShardingHash {
-
- static HashFunction hashFunc = Hashing.murmur3_128();
-
- public static long hashInt(int integer) {
- return hashFunc.newHasher().putInt(integer).hash().asLong();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
deleted file mode 100644
index 3443fc6..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-public class ShardingSliceBuilder {
-
- SliceBuilder[] builders;
-
- public ShardingSliceBuilder(TableRecordInfo info) {
- int sharding = info.getDescriptor().getSharding();
- builders = new SliceBuilder[sharding];
- for (short i = 0; i < sharding; i++) {
- builders[i] = new SliceBuilder(info, i);
- }
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- short shard = rec.getShard();
- return builders[shard].append(rec);
- }
-
- public List<Slice> close() {
- List<Slice> result = Lists.newArrayList();
- for (SliceBuilder builder : builders) {
- Slice slice = builder.close();
- if (slice != null)
- result.add(slice);
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
deleted file mode 100644
index 2a53864..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import org.roaringbitmap.RoaringBitmap;
-import org.roaringbitmap.IntIterator;
-
-/**
- * Within a partition (per timestampGranularity), records are further sliced
- * (per sliceLength) to fit into HBASE cell.
- *
- * @author yangli9
- */
-public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
-
- TableRecordInfoDigest info;
- int nColumns;
-
- short shard;
- long timestamp;
- int nRecords;
- ColumnValueContainer[] containers;
-
- public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
- this.info = digest;
- this.nColumns = digest.getColumnCount();
-
- this.shard = shard;
- this.timestamp = timestamp;
- this.nRecords = containers[0].getSize();
- this.containers = containers;
-
- assert nColumns == containers.length;
- for (int i = 0; i < nColumns; i++) {
- assert nRecords == containers[i].getSize();
- }
- }
-
- public int getRecordCount() {
- return this.nRecords;
- }
-
- public short getShard() {
- return shard;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ColumnValueContainer[] getColumnValueContainers() {
- return containers;
- }
-
- public ColumnValueContainer getColumnValueContainer(int col) {
- return containers[col];
- }
-
- public Iterator<RawTableRecord> iterateWithBitmap(final RoaringBitmap resultBitMap) {
- if (resultBitMap == null) {
- return this.iterator();
- } else {
- final RawTableRecord rec = info.createTableRecordBytes();
- final ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- return new Iterator<RawTableRecord>() {
- IntIterator iter = resultBitMap.getIntIterator();
-
- @Override
- public boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public RawTableRecord next() {
- int i = iter.next();
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
- }
-
- @Override
- public Iterator<RawTableRecord> iterator() {
- return new Iterator<RawTableRecord>() {
- int i = 0;
- RawTableRecord rec = info.createTableRecordBytes();
- ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- @Override
- public boolean hasNext() {
- return i < nRecords;
- }
-
- @Override
- public RawTableRecord next() {
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- i++;
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((info == null) ? 0 : info.hashCode());
- result = prime * result + shard;
- result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Slice other = (Slice) obj;
- if (info == null) {
- if (other.info != null)
- return false;
- } else if (!info.equals(other.info))
- return false;
- if (shard != other.shard)
- return false;
- if (timestamp != other.timestamp)
- return false;
- return true;
- }
-
- @Override
- public int compareTo(Slice o) {
- int comp = this.shard - o.shard;
- if (comp != 0)
- return comp;
-
- comp = (int) (this.timestamp - o.timestamp);
- return comp;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
deleted file mode 100644
index d794708..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-/**
- * @author yangli9
- */
-public class SliceBuilder {
-
- TableRecordInfo info;
- private int nColumns;
- int nRecordsCap;
-
- short shard;
- long sliceTimestamp;
- int nRecords;
- private ColumnValueContainer[] containers;
-
- transient ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- public SliceBuilder(TableRecordInfo info, short shard) {
- this.info = info;
- this.nColumns = info.getDigest().getColumnCount();
- this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize());
-
- this.shard = shard;
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- this.containers = null;
-
- doneSlice(); // init containers
- }
-
- private Slice doneSlice() {
- Slice r = null;
- if (nRecords > 0) {
- for (int i = 0; i < nColumns; i++) {
- containers[i].closeForChange();
- }
- r = new Slice(info.getDigest(), shard, sliceTimestamp, containers);
- }
-
- // reset for next slice
- nRecords = 0;
- containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getValueColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
- for (int i : info.getDescriptor().getMetricsColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
-
- return r;
-
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- if (rec.getShard() != shard)
- throw new IllegalStateException();
-
- Slice doneSlice = null;
-
- if (isFull()) {
- doneSlice = doneSlice();
- }
-
- if (nRecords == 0) {
- sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp());
- }
-
- nRecords++;
- for (int i = 0; i < nColumns; i++) {
- rec.getValueBytes(i, temp);
- containers[i].append(temp);
- }
-
- return doneSlice;
- }
-
- private long increaseSliceTimestamp(long timestamp) {
- if (timestamp < sliceTimestamp)
- throw new IllegalStateException();
-
- if (timestamp == sliceTimestamp)
- return ++timestamp; // ensure slice timestamp increases
- else
- return timestamp;
- }
-
- public Slice close() {
- Slice doneSlice = doneSlice();
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- return doneSlice;
- }
-
- private boolean isFull() {
- return nRecords >= nRecordsCap;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
deleted file mode 100644
index 09a9244..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import org.apache.commons.lang.ObjectUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.DateFormat;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9, honma
- * <p/>
- * TableRecord extends RawTableRecord by decorating it with a
- * TableRecordInfo
- */
-public class TableRecord implements Cloneable {
-
- private TableRecordInfo info;
- private RawTableRecord rawRecord;
-
- public TableRecord(RawTableRecord rawRecord, TableRecordInfo info) {
- this.info = info;
- this.rawRecord = rawRecord;
- }
-
- public TableRecord(TableRecord another) {
- this.info = another.info;
- this.rawRecord = (RawTableRecord) another.rawRecord.clone();
- }
-
- @Override
- public Object clone() {
- return new TableRecord(this);
- }
-
- public void reset() {
- rawRecord.reset();
- }
-
- public byte[] getBytes() {
- return rawRecord.getBytes();
- }
-
- public void setBytes(byte[] bytes, int offset, int length) {
- rawRecord.setBytes(bytes, offset, length);
- }
-
- public long getTimestamp() {
- String str = getValueString(info.getTimestampColumn());
- return DateFormat.stringToMillis(str);
- }
-
- public int length(int col) {
- return rawRecord.length(col);
- }
-
- public List<String> getOriginTableColumnValues() {
- List<String> ret = Lists.newArrayList();
- for (int i = 0; i < info.nColumns; ++i) {
- ret.add(getValueString(i));
- }
- return ret;
- }
-
- public void setValueString(int col, String value) {
- if (rawRecord.isMetric(col)) {
- LongWritable v = rawRecord.codec(col).valueOf(value);
- setValueMetrics(col, v);
- } else {
- int id = info.dict(col).getIdFromValue(value);
- rawRecord.setValueID(col, id);
- }
- }
-
- /**
- * get value of columns which belongs to the original table columns.
- * i.e. columns like min_xx, max_yy will never appear
- */
- public String getValueString(int col) {
- if (rawRecord.isMetric(col))
- return getValueMetric(col);
- else
- return info.dict(col).getValueFromId(rawRecord.getValueID(col));
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- rawRecord.getValueBytes(col, bytes);
- }
-
- private void setValueMetrics(int col, LongWritable value) {
- rawRecord.setValueMetrics(col, value);
- }
-
- private String getValueMetric(int col) {
- return rawRecord.getValueMetric(col);
- }
-
- public short getShard() {
- int timestampID = rawRecord.getValueID(info.getTimestampColumn());
- return (short) (Math.abs(ShardingHash.hashInt(timestampID)) % info.getDescriptor().getSharding());
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder("[");
- for (int col = 0; col < rawRecord.getColumnCount(); col++) {
- if (col > 0)
- buf.append(",");
- buf.append(getValueString(col));
- }
- buf.append("]");
- return buf.toString();
- }
-
- @Override
- public int hashCode() {
- if (rawRecord != null) {
- return rawRecord.hashCode();
- } else {
- return 0;
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecord other = (TableRecord) obj;
- return ObjectUtils.equals(other.rawRecord, this.rawRecord);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
deleted file mode 100644
index c41a70c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- * <p/>
- * TableRecordInfo stores application-aware knowledges, while
- * TableRecordInfoDigest only stores byte level knowleges
- */
-public class TableRecordInfo {
-
- final IISegment seg;
- final IIDesc desc;
- final int nColumns;
- final List<TblColRef> allColumns;
-
- final FixedLenMeasureCodec<?>[] measureSerializers;
- final Dictionary<?>[] dictionaries;
-
- final TableRecordInfoDigest digest;
-
- public TableRecordInfo(IISegment iiSegment) {
-
- seg = iiSegment;
- desc = seg.getIIInstance().getDescriptor();
- allColumns = desc.listAllColumns();
- nColumns = allColumns.size();
- dictionaries = new Dictionary<?>[nColumns];
- measureSerializers = new FixedLenMeasureCodec<?>[nColumns];
-
- DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig());
- int index = 0;
- for (TblColRef tblColRef : desc.listAllColumns()) {
- ColumnDesc col = tblColRef.getColumn();
- if (desc.isMetricsCol(index)) {
- measureSerializers[index] = FixedLenMeasureCodec.get(col.getType());
- } else {
- String dictPath = seg.getDictResPath(tblColRef);
- try {
- dictionaries[index] = dictMgr.getDictionary(dictPath);
- } catch (IOException e) {
- throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
- }
- }
- index++;
- }
-
- digest = createDigest();
- }
-
- public TableRecordInfoDigest getDigest() {
- return digest;
- }
-
- private TableRecordInfoDigest createDigest() {
- // isMetric
- boolean[] isMetric = new boolean[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- isMetric[i] = desc.isMetricsCol(i);
- }
-
- // lengths
- int[] lengths = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- lengths[i] = isMetric[i] ? measureSerializers[i].getLength() : dictionaries[i].getSizeOfId();
- }
-
- // dict max id
- int[] dictMaxIds = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- if (!isMetric[i])
- dictMaxIds[i] = dictionaries[i].getMaxId();
- }
-
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
-
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, measureSerializers);
- }
-
- public TableRecord createTableRecord() {
- return new TableRecord(digest.createTableRecordBytes(), this);
- }
-
- public IIDesc getDescriptor() {
- return desc;
- }
-
- public List<TblColRef> getColumns() {
- return allColumns;
- }
-
- public int findColumn(TblColRef col) {
- return desc.findColumn(col);
- }
-
- public int findFactTableColumn(String columnName) {
- if (columnName == null)
- return -1;
- for (int i = 0; i < allColumns.size(); ++i) {
- TblColRef tblColRef = allColumns.get(i);
- if (tblColRef.isSameAs(desc.getFactTableName(), columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- // dimensions go with dictionary
- @SuppressWarnings("unchecked")
- public Dictionary<String> dict(int col) {
- // yes, all dictionaries are string based
- return (Dictionary<String>) dictionaries[col];
- }
-
- public int getTimestampColumn() {
- return desc.getTimestampColumn();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((seg == null) ? 0 : seg.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecordInfo other = (TableRecordInfo) obj;
- if (seg == null) {
- if (other.seg != null)
- return false;
- } else if (!seg.equals(other.seg))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
deleted file mode 100644
index 0ed58b0..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.DataType;
-
-/**
- * Created by honma on 11/10/14.
- */
-public class TableRecordInfoDigest {
-
- private int nColumns;
- private int byteFormLen;
-
- private int[] offsets;// column offset in byte form row
- private int[] dictMaxIds;// max id for each of the dict
- private int[] lengths;// length of each encoded dict
- private boolean[] isMetric;// whether it's metric or dict
-
- protected FixedLenMeasureCodec<?>[] measureSerializers;
-
- public TableRecordInfoDigest(int nColumns, int byteFormLen, int[] offsets, int[] dictMaxIds, int[] lengths, boolean[] isMetric, FixedLenMeasureCodec<?>[] measureSerializers) {
- this.nColumns = nColumns;
- this.byteFormLen = byteFormLen;
- this.offsets = offsets;
- this.dictMaxIds = dictMaxIds;
- this.lengths = lengths;
- this.isMetric = isMetric;
- this.measureSerializers = measureSerializers;
- }
-
- public TableRecordInfoDigest() {
- }
-
- public int getByteFormLen() {
- return byteFormLen;
- }
-
- public boolean isMetrics(int col) {
- return isMetric[col];
- }
-
- public int getColumnCount() {
- return nColumns;
- }
-
- public int offset(int col) {
- return offsets[col];
- }
-
- public int length(int col) {
- return lengths[col];
- }
-
- public int getMaxID(int col) {
- return dictMaxIds[col];
- }
-
- public int getMetricCount() {
- int ret = 0;
- for (int i = 0; i < nColumns; ++i) {
- if (isMetrics(i)) {
- ret++;
- }
- }
- return ret;
- }
-
- public RawTableRecord createTableRecordBytes() {
- return new RawTableRecord(this);
- }
-
- // metrics go with fixed-len codec
- @SuppressWarnings("unchecked")
- public FixedLenMeasureCodec<LongWritable> codec(int col) {
- // yes, all metrics are long currently
- return (FixedLenMeasureCodec<LongWritable>) measureSerializers[col];
- }
-
- public static byte[] serialize(TableRecordInfoDigest o) {
- ByteBuffer buf = ByteBuffer.allocate(Serializer.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static TableRecordInfoDigest deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- public static TableRecordInfoDigest deserialize(ByteBuffer buffer) {
- return serializer.deserialize(buffer);
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<TableRecordInfoDigest> {
-
- @Override
- public void serialize(TableRecordInfoDigest value, ByteBuffer out) {
- BytesUtil.writeVInt(value.nColumns, out);
- BytesUtil.writeVInt(value.byteFormLen, out);
- BytesUtil.writeIntArray(value.offsets, out);
- BytesUtil.writeIntArray(value.dictMaxIds, out);
- BytesUtil.writeIntArray(value.lengths, out);
- BytesUtil.writeBooleanArray(value.isMetric, out);
-
- for (int i = 0; i < value.measureSerializers.length; ++i) {
- if (value.isMetrics(i)) {
- BytesUtil.writeAsciiString(value.measureSerializers[i].getDataType().toString(), out);
- } else {
- BytesUtil.writeAsciiString(null, out);
- }
- }
- }
-
- @Override
- public TableRecordInfoDigest deserialize(ByteBuffer in) {
- TableRecordInfoDigest result = new TableRecordInfoDigest();
- result.nColumns = BytesUtil.readVInt(in);
- result.byteFormLen = BytesUtil.readVInt(in);
- result.offsets = BytesUtil.readIntArray(in);
- result.dictMaxIds = BytesUtil.readIntArray(in);
- result.lengths = BytesUtil.readIntArray(in);
- result.isMetric = BytesUtil.readBooleanArray(in);
-
- result.measureSerializers = new FixedLenMeasureCodec<?>[result.nColumns];
- for (int i = 0; i < result.nColumns; ++i) {
- String typeStr = BytesUtil.readAsciiString(in);
- if (typeStr == null) {
- result.measureSerializers[i] = null;
- } else {
- result.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(typeStr));
- }
- }
-
- return result;
- }
-
- }
-}
[49/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
deleted file mode 100644
index aed7c27..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ /dev/null
@@ -1,5028 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.calcite.sql2rel;
-
-import org.apache.calcite.avatica.util.Spaces;
-import org.apache.calcite.linq4j.Ord;
-import org.apache.calcite.plan.Convention;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptSamplingParameters;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.prepare.Prepare;
-import org.apache.calcite.prepare.RelOptTableImpl;
-import org.apache.calcite.rel.RelCollation;
-import org.apache.calcite.rel.RelCollationTraitDef;
-import org.apache.calcite.rel.RelCollations;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelRoot;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Collect;
-import org.apache.calcite.rel.core.CorrelationId;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.JoinInfo;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.core.Sample;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.Uncollect;
-import org.apache.calcite.rel.logical.LogicalAggregate;
-import org.apache.calcite.rel.logical.LogicalCorrelate;
-import org.apache.calcite.rel.logical.LogicalFilter;
-import org.apache.calcite.rel.logical.LogicalIntersect;
-import org.apache.calcite.rel.logical.LogicalJoin;
-import org.apache.calcite.rel.logical.LogicalMinus;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rel.logical.LogicalSort;
-import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
-import org.apache.calcite.rel.logical.LogicalTableModify;
-import org.apache.calcite.rel.logical.LogicalTableScan;
-import org.apache.calcite.rel.logical.LogicalUnion;
-import org.apache.calcite.rel.logical.LogicalValues;
-import org.apache.calcite.rel.metadata.RelColumnMapping;
-import org.apache.calcite.rel.stream.Delta;
-import org.apache.calcite.rel.stream.LogicalDelta;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexCallBinding;
-import org.apache.calcite.rex.RexCorrelVariable;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexFieldAccess;
-import org.apache.calcite.rex.RexFieldCollation;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexRangeRef;
-import org.apache.calcite.rex.RexShuttle;
-import org.apache.calcite.rex.RexSubQuery;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexWindowBound;
-import org.apache.calcite.schema.ModifiableTable;
-import org.apache.calcite.schema.ModifiableView;
-import org.apache.calcite.schema.Table;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.sql.JoinConditionType;
-import org.apache.calcite.sql.JoinType;
-import org.apache.calcite.sql.SemiJoinType;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlBasicCall;
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlCallBinding;
-import org.apache.calcite.sql.SqlDataTypeSpec;
-import org.apache.calcite.sql.SqlDelete;
-import org.apache.calcite.sql.SqlDynamicParam;
-import org.apache.calcite.sql.SqlExplainLevel;
-import org.apache.calcite.sql.SqlFunction;
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlInsert;
-import org.apache.calcite.sql.SqlIntervalQualifier;
-import org.apache.calcite.sql.SqlJoin;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlLiteral;
-import org.apache.calcite.sql.SqlMerge;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.SqlNumericLiteral;
-import org.apache.calcite.sql.SqlOperator;
-import org.apache.calcite.sql.SqlOperatorTable;
-import org.apache.calcite.sql.SqlOrderBy;
-import org.apache.calcite.sql.SqlSampleSpec;
-import org.apache.calcite.sql.SqlSelect;
-import org.apache.calcite.sql.SqlSelectKeyword;
-import org.apache.calcite.sql.SqlSetOperator;
-import org.apache.calcite.sql.SqlUnnestOperator;
-import org.apache.calcite.sql.SqlUpdate;
-import org.apache.calcite.sql.SqlUtil;
-import org.apache.calcite.sql.SqlValuesOperator;
-import org.apache.calcite.sql.SqlWindow;
-import org.apache.calcite.sql.SqlWith;
-import org.apache.calcite.sql.SqlWithItem;
-import org.apache.calcite.sql.fun.SqlCountAggFunction;
-import org.apache.calcite.sql.fun.SqlInOperator;
-import org.apache.calcite.sql.fun.SqlRowOperator;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.parser.SqlParserPos;
-import org.apache.calcite.sql.type.SqlReturnTypeInference;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.sql.type.SqlTypeUtil;
-import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
-import org.apache.calcite.sql.util.SqlBasicVisitor;
-import org.apache.calcite.sql.util.SqlVisitor;
-import org.apache.calcite.sql.validate.AggregatingSelectScope;
-import org.apache.calcite.sql.validate.CollectNamespace;
-import org.apache.calcite.sql.validate.DelegatingScope;
-import org.apache.calcite.sql.validate.ListScope;
-import org.apache.calcite.sql.validate.ParameterScope;
-import org.apache.calcite.sql.validate.SelectScope;
-import org.apache.calcite.sql.validate.SqlMonotonicity;
-import org.apache.calcite.sql.validate.SqlQualified;
-import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
-import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
-import org.apache.calcite.sql.validate.SqlValidator;
-import org.apache.calcite.sql.validate.SqlValidatorImpl;
-import org.apache.calcite.sql.validate.SqlValidatorNamespace;
-import org.apache.calcite.sql.validate.SqlValidatorScope;
-import org.apache.calcite.sql.validate.SqlValidatorUtil;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Litmus;
-import org.apache.calcite.util.NlsString;
-import org.apache.calcite.util.NumberUtil;
-import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Util;
-import org.apache.calcite.util.trace.CalciteTrace;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-/*
- * OVERRIDE POINT:
- * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
- * - isTrimUnusedFields(), override to false
- * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
- */
-
-/**
- * Converts a SQL parse tree (consisting of
- * {@link org.apache.calcite.sql.SqlNode} objects) into a relational algebra
- * expression (consisting of {@link org.apache.calcite.rel.RelNode} objects).
- *
- * <p>The public entry points are: {@link #convertQuery},
- * {@link #convertExpression(SqlNode)}.
- */
-public class SqlToRelConverter {
- //~ Static fields/initializers ---------------------------------------------
-
- protected static final Logger SQL2REL_LOGGER =
- CalciteTrace.getSqlToRelTracer();
-
- private static final BigDecimal TWO = BigDecimal.valueOf(2L);
-
- /** Size of the smallest IN list that will be converted to a semijoin to a
- * static table. */
- public static final int IN_SUBQUERY_THRESHOLD = 20;
-
- //~ Instance fields --------------------------------------------------------
-
- protected final SqlValidator validator;
- protected final RexBuilder rexBuilder;
- protected final Prepare.CatalogReader catalogReader;
- protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
- protected final List<RelNode> leaves = new ArrayList<>();
- private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
- private final SqlOperatorTable opTab;
- private boolean shouldConvertTableAccess;
- protected final RelDataTypeFactory typeFactory;
- private final SqlNodeToRexConverter exprConverter;
- private boolean decorrelationEnabled;
- private boolean trimUnusedFields;
- private boolean shouldCreateValuesRel;
- private boolean isExplain;
- private int nDynamicParamsInExplain;
-
- /**
- * Fields used in name resolution for correlated subqueries.
- */
- private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred =
- new HashMap<>();
-
- /**
- * Stack of names of datasets requested by the <code>
- * TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
- */
- private final Deque<String> datasetStack = new ArrayDeque<>();
-
- /**
- * Mapping of non-correlated subqueries that have been converted to their
- * equivalent constants. Used to avoid re-evaluating the subquery if it's
- * already been evaluated.
- */
- private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
- new HashMap<>();
-
- public final RelOptTable.ViewExpander viewExpander;
-
- /** Whether to expand sub-queries. If false, each sub-query becomes a
- * {@link org.apache.calcite.rex.RexSubQuery}. */
- private boolean expand = true;
-
- //~ Constructors -----------------------------------------------------------
- /**
- * Creates a converter.
- *
- * @param viewExpander Preparing statement
- * @param validator Validator
- * @param catalogReader Schema
- * @param planner Planner
- * @param rexBuilder Rex builder
- * @param convertletTable Expression converter
- */
- @Deprecated // will be removed before 2.0
- public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptPlanner planner,
- RexBuilder rexBuilder,
- SqlRexConvertletTable convertletTable) {
- this(viewExpander, validator, catalogReader,
- RelOptCluster.create(planner, rexBuilder), convertletTable);
- }
-
- /* Creates a converter. */
- public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptCluster cluster,
- SqlRexConvertletTable convertletTable) {
- this.viewExpander = viewExpander;
- this.opTab =
- (validator
- == null) ? SqlStdOperatorTable.instance()
- : validator.getOperatorTable();
- this.validator = validator;
- this.catalogReader = catalogReader;
- this.defaultValueFactory = new NullDefaultValueFactory();
- this.subqueryConverter = new NoOpSubqueryConverter();
- this.rexBuilder = cluster.getRexBuilder();
- this.typeFactory = rexBuilder.getTypeFactory();
- this.cluster = Preconditions.checkNotNull(cluster);
- this.shouldConvertTableAccess = true;
- this.exprConverter =
- new SqlNodeToRexConverterImpl(convertletTable);
- decorrelationEnabled = true;
- trimUnusedFields = false;
- shouldCreateValuesRel = true;
- isExplain = false;
- nDynamicParamsInExplain = 0;
- }
-
- //~ Methods ----------------------------------------------------------------
-
- /**
- * @return the RelOptCluster in use.
- */
- public RelOptCluster getCluster() {
- return cluster;
- }
-
- /**
- * Returns the row-expression builder.
- */
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- /**
- * Returns the number of dynamic parameters encountered during translation;
- * this must only be called after {@link #convertQuery}.
- *
- * @return number of dynamic parameters
- */
- public int getDynamicParamCount() {
- return dynamicParamSqlNodes.size();
- }
-
- /**
- * Returns the type inferred for a dynamic parameter.
- *
- * @param index 0-based index of dynamic parameter
- * @return inferred type, never null
- */
- public RelDataType getDynamicParamType(int index) {
- SqlNode sqlNode = dynamicParamSqlNodes.get(index);
- if (sqlNode == null) {
- throw Util.needToImplement("dynamic param type inference");
- }
- return validator.getValidatedNodeType(sqlNode);
- }
-
- /**
- * Returns the current count of the number of dynamic parameters in an
- * EXPLAIN PLAN statement.
- *
- * @param increment if true, increment the count
- * @return the current count before the optional increment
- */
- public int getDynamicParamCountInExplain(boolean increment) {
- int retVal = nDynamicParamsInExplain;
- if (increment) {
- ++nDynamicParamsInExplain;
- }
- return retVal;
- }
-
- /**
- * @return mapping of non-correlated subqueries that have been converted to
- * the constants that they evaluate to
- */
- public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
- return mapConvertedNonCorrSubqs;
- }
-
- /**
- * Adds to the current map of non-correlated converted subqueries the
- * elements from another map that contains non-correlated subqueries that
- * have been converted by another SqlToRelConverter.
- *
- * @param alreadyConvertedNonCorrSubqs the other map
- */
- public void addConvertedNonCorrSubqs(
- Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
- mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
- }
-
- /**
- * Set a new DefaultValueFactory. To have any effect, this must be called
- * before any convert method.
- *
- * @param factory new DefaultValueFactory
- */
- public void setDefaultValueFactory(DefaultValueFactory factory) {
- defaultValueFactory = factory;
- }
-
- /**
- * Sets a new SubqueryConverter. To have any effect, this must be called
- * before any convert method.
- *
- * @param converter new SubqueryConverter
- */
- public void setSubqueryConverter(SubqueryConverter converter) {
- subqueryConverter = converter;
- }
-
- /**
- * Indicates that the current statement is part of an EXPLAIN PLAN statement
- *
- * @param nDynamicParams number of dynamic parameters in the statement
- */
- public void setIsExplain(int nDynamicParams) {
- isExplain = true;
- nDynamicParamsInExplain = nDynamicParams;
- }
-
- /**
- * Controls whether table access references are converted to physical rels
- * immediately. The optimizer doesn't like leaf rels to have
- * {@link Convention#NONE}. However, if we are doing further conversion
- * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
- * defer conversion. To have any effect, this must be called before any
- * convert method.
- *
- * @param enabled true for immediate conversion (the default); false to
- * generate logical LogicalTableScan instances
- */
- public void enableTableAccessConversion(boolean enabled) {
- shouldConvertTableAccess = enabled;
- }
-
- /**
- * Controls whether instances of
- * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
- * may not be supported by all physical implementations. To have any effect,
- * this must be called before any convert method.
- *
- * @param enabled true to allow LogicalValues to be generated (the default);
- * false to force substitution of Project+OneRow instead
- */
- public void enableValuesRelCreation(boolean enabled) {
- shouldCreateValuesRel = enabled;
- }
-
- private void checkConvertedType(SqlNode query, RelNode result) {
- if (query.isA(SqlKind.DML)) {
- return;
- }
- // Verify that conversion from SQL to relational algebra did
- // not perturb any type information. (We can't do this if the
- // SQL statement is something like an INSERT which has no
- // validator type information associated with its result,
- // hence the namespace check above.)
- final List<RelDataTypeField> validatedFields =
- validator.getValidatedNodeType(query).getFieldList();
- final RelDataType validatedRowType =
- validator.getTypeFactory().createStructType(
- Pair.right(validatedFields),
- SqlValidatorUtil.uniquify(Pair.left(validatedFields)));
-
- final List<RelDataTypeField> convertedFields =
- result.getRowType().getFieldList().subList(0, validatedFields.size());
- final RelDataType convertedRowType =
- validator.getTypeFactory().createStructType(convertedFields);
-
- if (!RelOptUtil.equal("validated row type", validatedRowType,
- "converted row type", convertedRowType, Litmus.IGNORE)) {
- throw new AssertionError("Conversion to relational algebra failed to "
- + "preserve datatypes:\n"
- + "validated type:\n"
- + validatedRowType.getFullTypeString()
- + "\nconverted type:\n"
- + convertedRowType.getFullTypeString()
- + "\nrel:\n"
- + RelOptUtil.toString(result));
- }
- }
-
- public RelNode flattenTypes(
- RelNode rootRel,
- boolean restructure) {
- RelStructuredTypeFlattener typeFlattener =
- new RelStructuredTypeFlattener(rexBuilder, createToRelContext());
- return typeFlattener.rewrite(rootRel, restructure);
- }
-
- /**
- * If subquery is correlated and decorrelation is enabled, performs
- * decorrelation.
- *
- * @param query Query
- * @param rootRel Root relational expression
- * @return New root relational expression after decorrelation
- */
- public RelNode decorrelate(SqlNode query, RelNode rootRel) {
- if (!enableDecorrelation()) {
- return rootRel;
- }
- final RelNode result = decorrelateQuery(rootRel);
- if (result != rootRel) {
- checkConvertedType(query, result);
- }
- return result;
- }
-
- /**
- * Walks over a tree of relational expressions, replacing each
- * {@link RelNode} with a 'slimmed down' relational expression that projects
- * only the fields required by its consumer.
- *
- * <p>This may make things easier for the optimizer, by removing crud that
- * would expand the search space, but is difficult for the optimizer itself
- * to do it, because optimizer rules must preserve the number and type of
- * fields. Hence, this transform that operates on the entire tree, similar
- * to the {@link RelStructuredTypeFlattener type-flattening transform}.
- *
- * <p>Currently this functionality is disabled in farrago/luciddb; the
- * default implementation of this method does nothing.
- *
- * @param ordered Whether the relational expression must produce results in
- * a particular order (typically because it has an ORDER BY at top level)
- * @param rootRel Relational expression that is at the root of the tree
- * @return Trimmed relational expression
- */
- public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) {
- // Trim fields that are not used by their consumer.
- if (isTrimUnusedFields()) {
- final RelFieldTrimmer trimmer = newFieldTrimmer();
- final List<RelCollation> collations =
- rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
- rootRel = trimmer.trim(rootRel);
- if (!ordered
- && collations != null
- && !collations.isEmpty()
- && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
- final RelTraitSet traitSet = rootRel.getTraitSet()
- .replace(RelCollationTraitDef.INSTANCE, collations);
- rootRel = rootRel.copy(traitSet, rootRel.getInputs());
- }
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after trimming unused fields",
- rootRel,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
- }
- return rootRel;
- }
-
- /**
- * Creates a RelFieldTrimmer.
- *
- * @return Field trimmer
- */
- protected RelFieldTrimmer newFieldTrimmer() {
- final RelBuilder relBuilder =
- RelFactories.LOGICAL_BUILDER.create(cluster, null);
- return new RelFieldTrimmer(validator, relBuilder);
- }
-
- /**
- * Converts an unvalidated query's parse tree into a relational expression.
- *
- * @param query Query to convert
- * @param needsValidation Whether to validate the query before converting;
- * <code>false</code> if the query has already been
- * validated.
- * @param top Whether the query is top-level, say if its result
- * will become a JDBC result set; <code>false</code> if
- * the query will be part of a view.
- */
- public RelRoot convertQuery(
- SqlNode query,
- final boolean needsValidation,
- final boolean top) {
- if (needsValidation) {
- query = validator.validate(query);
- }
-
- RelNode result = convertQueryRecursive(query, top, null).rel;
- if (top) {
- if (isStream(query)) {
- result = new LogicalDelta(cluster, result.getTraitSet(), result);
- }
- }
- RelCollation collation = RelCollations.EMPTY;
- if (!query.isA(SqlKind.DML)) {
- if (isOrdered(query)) {
- collation = requiredCollation(result);
- }
- }
- checkConvertedType(query, result);
-
- boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE);
- if (dumpPlan) {
- SQL2REL_LOGGER.fine(
- RelOptUtil.dumpPlan(
- "Plan after converting SqlNode to RelNode",
- result,
- false,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
- }
-
- final RelDataType validatedRowType = validator.getValidatedNodeType(query);
- return RelRoot.of(result, validatedRowType, query.getKind())
- .withCollation(collation);
- }
-
- private static boolean isStream(SqlNode query) {
- return query instanceof SqlSelect
- && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
- }
-
- public static boolean isOrdered(SqlNode query) {
- switch (query.getKind()) {
- case SELECT:
- return ((SqlSelect) query).getOrderList() != null
- && ((SqlSelect) query).getOrderList().size() > 0;
- case WITH:
- return isOrdered(((SqlWith) query).body);
- case ORDER_BY:
- return ((SqlOrderBy) query).orderList.size() > 0;
- default:
- return false;
- }
- }
-
- private RelCollation requiredCollation(RelNode r) {
- if (r instanceof Sort) {
- return ((Sort) r).collation;
- }
- if (r instanceof Project) {
- return requiredCollation(((Project) r).getInput());
- }
- if (r instanceof Delta) {
- return requiredCollation(((Delta) r).getInput());
- }
- throw new AssertionError();
- }
-
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertSelect(SqlSelect select, boolean top) {
- final SqlValidatorScope selectScope = validator.getWhereScope(select);
- final Blackboard bb = createBlackboard(selectScope, null, top);
- convertSelectImpl(bb, select);
- return bb.root;
- }
-
- /**
- * Factory method for creating translation workspace.
- */
- protected Blackboard createBlackboard(SqlValidatorScope scope,
- Map<String, RexNode> nameToNodeMap, boolean top) {
- return new Blackboard(scope, nameToNodeMap, top);
- }
-
- /**
- * Implementation of {@link #convertSelect(SqlSelect, boolean)};
- * derived class may override.
- */
- protected void convertSelectImpl(
- final Blackboard bb,
- SqlSelect select) {
- convertFrom(
- bb,
- select.getFrom());
- convertWhere(
- bb,
- select.getWhere());
-
- final List<SqlNode> orderExprList = new ArrayList<>();
- final List<RelFieldCollation> collationList = new ArrayList<>();
- gatherOrderExprs(
- bb,
- select,
- select.getOrderList(),
- orderExprList,
- collationList);
- final RelCollation collation =
- cluster.traitSet().canonize(RelCollations.of(collationList));
-
- if (validator.isAggregate(select)) {
- convertAgg(
- bb,
- select,
- orderExprList);
- } else {
- convertSelectList(
- bb,
- select,
- orderExprList);
- }
-
- if (select.isDistinct()) {
- distinctify(bb, true);
- }
- convertOrder(
- select, bb, collation, orderExprList, select.getOffset(),
- select.getFetch());
- bb.setRoot(bb.root, true);
- }
-
- /**
- * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds
- * a relational expression to make the results unique.
- *
- * <p>If the SELECT clause contains duplicate expressions, adds
- * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are
- * grouping on the minimal set of keys. The performance gain isn't huge, but
- * it is difficult to detect these duplicate expressions later.
- *
- * @param bb Blackboard
- * @param checkForDupExprs Check for duplicate expressions
- */
- private void distinctify(
- Blackboard bb,
- boolean checkForDupExprs) {
- // Look for duplicate expressions in the project.
- // Say we have 'select x, y, x, z'.
- // Then dups will be {[2, 0]}
- // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
- RelNode rel = bb.root;
- if (checkForDupExprs && (rel instanceof LogicalProject)) {
- LogicalProject project = (LogicalProject) rel;
- final List<RexNode> projectExprs = project.getProjects();
- final List<Integer> origins = new ArrayList<>();
- int dupCount = 0;
- for (int i = 0; i < projectExprs.size(); i++) {
- int x = findExpr(projectExprs.get(i), projectExprs, i);
- if (x >= 0) {
- origins.add(x);
- ++dupCount;
- } else {
- origins.add(i);
- }
- }
- if (dupCount == 0) {
- distinctify(bb, false);
- return;
- }
-
- final Map<Integer, Integer> squished = Maps.newHashMap();
- final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
- final List<Pair<RexNode, String>> newProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- if (origins.get(i) == i) {
- squished.put(i, newProjects.size());
- newProjects.add(RexInputRef.of2(i, fields));
- }
- }
- rel =
- LogicalProject.create(rel, Pair.left(newProjects),
- Pair.right(newProjects));
- bb.root = rel;
- distinctify(bb, false);
- rel = bb.root;
-
- // Create the expressions to reverse the mapping.
- // Project($0, $1, $0, $2).
- final List<Pair<RexNode, String>> undoProjects = Lists.newArrayList();
- for (int i = 0; i < fields.size(); i++) {
- final int origin = origins.get(i);
- RelDataTypeField field = fields.get(i);
- undoProjects.add(
- Pair.of(
- (RexNode) new RexInputRef(
- squished.get(origin), field.getType()),
- field.getName()));
- }
-
- rel =
- LogicalProject.create(rel, Pair.left(undoProjects),
- Pair.right(undoProjects));
- bb.setRoot(
- rel,
- false);
-
- return;
- }
-
- // Usual case: all of the expressions in the SELECT clause are
- // different.
- final ImmutableBitSet groupSet =
- ImmutableBitSet.range(rel.getRowType().getFieldCount());
- rel =
- createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
- ImmutableList.<AggregateCall>of());
-
- bb.setRoot(
- rel,
- false);
- }
-
- private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
- for (int i = 0; i < count; i++) {
- RexNode expr = exprs.get(i);
- if (expr.toString().equals(seek.toString())) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Converts a query's ORDER BY clause, if any.
- *
- * @param select Query
- * @param bb Blackboard
- * @param collation Collation list
- * @param orderExprList Method populates this list with orderBy expressions
- * not present in selectList
- * @param offset Expression for number of rows to discard before
- * returning first row
- * @param fetch Expression for number of rows to fetch
- */
- protected void convertOrder(
- SqlSelect select,
- Blackboard bb,
- RelCollation collation,
- List<SqlNode> orderExprList,
- SqlNode offset,
- SqlNode fetch) {
- if (select.getOrderList() == null
- || select.getOrderList().getList().isEmpty()) {
- assert collation.getFieldCollations().isEmpty();
- if ((offset == null
- || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
- && fetch == null) {
- return;
- }
- }
-
- // Create a sorter using the previously constructed collations.
- bb.setRoot(
- LogicalSort.create(bb.root, collation,
- offset == null ? null : convertExpression(offset),
- fetch == null ? null : convertExpression(fetch)),
- false);
-
- // If extra expressions were added to the project list for sorting,
- // add another project to remove them. But make the collation empty, because
- // we can't represent the real collation.
- //
- // If it is the top node, use the real collation, but don't trim fields.
- if (orderExprList.size() > 0 && !bb.top) {
- final List<RexNode> exprs = new ArrayList<>();
- final RelDataType rowType = bb.root.getRowType();
- final int fieldCount =
- rowType.getFieldCount() - orderExprList.size();
- for (int i = 0; i < fieldCount; i++) {
- exprs.add(rexBuilder.makeInputRef(bb.root, i));
- }
- bb.setRoot(
- LogicalProject.create(bb.root, exprs,
- rowType.getFieldNames().subList(0, fieldCount)),
- false);
- }
- }
-
- /**
- * Returns whether a given node contains a {@link SqlInOperator}.
- *
- * @param node a RexNode tree
- */
- private static boolean containsInOperator(
- SqlNode node) {
- try {
- SqlVisitor<Void> visitor =
- new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
- }
- };
- node.accept(visitor);
- return false;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return true;
- }
- }
-
- /**
- * Push down all the NOT logical operators into any IN/NOT IN operators.
- *
- * @param sqlNode the root node from which to look for NOT operators
- * @return the transformed SqlNode representation with NOT pushed down.
- */
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
- if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
- SqlCall sqlCall = (SqlCall) sqlNode;
- if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
- || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
- SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
- for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
- }
- return sqlNode;
- } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode childNode = sqlCall.operand(0);
- assert childNode instanceof SqlCall;
- SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
- if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
- SqlNode[] andOperands = childSqlCall.getOperands();
- SqlNode[] orOperands = new SqlNode[andOperands.length];
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- andOperands[i]);
- }
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
- }
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
- orOperands[0], orOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
- SqlNode[] orOperands = childSqlCall.getOperands();
- SqlNode[] andOperands = new SqlNode[orOperands.length];
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] =
- SqlStdOperatorTable.NOT.createCall(
- SqlParserPos.ZERO,
- orOperands[i]);
- }
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
- }
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- andOperands[0], andOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode[] notOperands = childSqlCall.getOperands();
- assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
- } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
- SqlNode[] inOperands = childSqlCall.getOperands();
- SqlInOperator inOp =
- (SqlInOperator) childSqlCall.getOperator();
- if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- } else {
- return SqlStdOperatorTable.NOT_IN.createCall(
- SqlParserPos.ZERO,
- inOperands[0],
- inOperands[1]);
- }
- } else {
- // childSqlCall is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // sqlNode is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // tree rooted at sqlNode does not contain inOperator
- return sqlNode;
- }
- }
-
- /**
- * Converts a WHERE clause.
- *
- * @param bb Blackboard
- * @param where WHERE clause, may be null
- */
- private void convertWhere(
- final Blackboard bb,
- final SqlNode where) {
- if (where == null) {
- return;
- }
- SqlNode newWhere = pushDownNotForIn(where);
- replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- final RexNode convertedWhere = bb.convertExpression(newWhere);
-
- // only allocate filter if the condition is not TRUE
- if (convertedWhere.isAlwaysTrue()) {
- return;
- }
-
- final RelNode filter = RelOptUtil.createFilter(bb.root, convertedWhere);
- final RelNode r;
- final CorrelationUse p = getCorrelationUse(bb, filter);
- if (p != null) {
- assert p.r instanceof Filter;
- Filter f = (Filter) p.r;
- r = LogicalFilter.create(f.getInput(), f.getCondition(),
- ImmutableSet.of(p.id));
- } else {
- r = filter;
- }
-
- bb.setRoot(r, false);
- }
-
- private void replaceSubqueries(
- final Blackboard bb,
- final SqlNode expr,
- RelOptUtil.Logic logic) {
- findSubqueries(bb, expr, logic, false);
- for (SubQuery node : bb.subqueryList) {
- substituteSubquery(bb, node);
- }
- }
-
- private void substituteSubquery(Blackboard bb, SubQuery subQuery) {
- final RexNode expr = subQuery.expr;
- if (expr != null) {
- // Already done.
- return;
- }
-
- final SqlBasicCall call;
- final RelNode rel;
- final SqlNode query;
- final Pair<RelNode, Boolean> converted;
- switch (subQuery.node.getKind()) {
- case CURSOR:
- convertCursor(bb, subQuery);
- return;
-
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case ARRAY_QUERY_CONSTRUCTOR:
- rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
- subQuery.expr = bb.register(rel, JoinRelType.INNER);
- return;
-
- case IN:
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(1);
- if (!expand && !(query instanceof SqlNodeList)) {
- return;
- }
- final SqlNode leftKeyNode = call.operand(0);
-
- final List<RexNode> leftKeys;
- switch (leftKeyNode.getKind()) {
- case ROW:
- leftKeys = Lists.newArrayList();
- for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
- leftKeys.add(bb.convertExpression(sqlExpr));
- }
- break;
- default:
- leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
- }
-
- final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn();
- if (query instanceof SqlNodeList) {
- SqlNodeList valueList = (SqlNodeList) query;
- if (!containsNullLiteral(valueList)
- && valueList.size() < getInSubqueryThreshold()) {
- // We're under the threshold, so convert to OR.
- subQuery.expr =
- convertInToOr(
- bb,
- leftKeys,
- valueList,
- isNotIn);
- return;
- }
-
- // Otherwise, let convertExists translate
- // values list into an inline table for the
- // reference to Q below.
- }
-
- // Project out the search columns from the left side
-
- // Q1:
- // "select from emp where emp.deptno in (select col1 from T)"
- //
- // is converted to
- //
- // "select from
- // emp inner join (select distinct col1 from T)) q
- // on emp.deptno = q.col1
- //
- // Q2:
- // "select from emp where emp.deptno not in (Q)"
- //
- // is converted to
- //
- // "select from
- // emp left outer join (select distinct col1, TRUE from T) q
- // on emp.deptno = q.col1
- // where emp.deptno <> null
- // and q.indicator <> TRUE"
- //
- final boolean outerJoin = bb.subqueryNeedsOuterJoin
- || isNotIn
- || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
- final RelDataType targetRowType =
- SqlTypeUtil.promoteToRowType(typeFactory,
- validator.getValidatedNodeType(leftKeyNode), null);
- converted =
- convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic,
- outerJoin, targetRowType);
- if (converted.right) {
- // Generate
- // emp CROSS JOIN (SELECT COUNT(*) AS c,
- // COUNT(deptno) AS ck FROM dept)
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RelNode seek = converted.left.getInput(0); // fragile
- final int keyCount = leftKeys.size();
- final List<Integer> args = ImmutableIntList.range(0, keyCount);
- LogicalAggregate aggregate =
- LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
- ImmutableList.of(
- AggregateCall.create(SqlStdOperatorTable.COUNT, false,
- ImmutableList.<Integer>of(), -1, longType, null),
- AggregateCall.create(SqlStdOperatorTable.COUNT, false,
- args, -1, longType, null)));
- LogicalJoin join =
- LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
- ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
- bb.setRoot(join, false);
- }
- RexNode rex =
- bb.register(converted.left,
- outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
-
- subQuery.expr = translateIn(subQuery, bb.root, rex);
- if (isNotIn) {
- subQuery.expr =
- rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
- }
- return;
-
- case EXISTS:
- // "select from emp where exists (select a from T)"
- //
- // is converted to the following if the subquery is correlated:
- //
- // "select from emp left outer join (select AGG_TRUE() as indicator
- // from T group by corr_var) q where q.indicator is true"
- //
- // If there is no correlation, the expression is replaced with a
- // boolean indicating whether the subquery returned 0 or >= 1 row.
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(0);
- if (!expand) {
- return;
- }
- converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS,
- subQuery.logic, true, null);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) {
- return;
- }
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- case SCALAR_QUERY:
- // Convert the subquery. If it's non-correlated, convert it
- // to a constant expression.
- if (!expand) {
- return;
- }
- call = (SqlBasicCall) subQuery.node;
- query = call.operand(0);
- converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true, null);
- assert !converted.right;
- if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) {
- return;
- }
- rel = convertToSingleValueSubq(query, converted.left);
- subQuery.expr = bb.register(rel, JoinRelType.LEFT);
- return;
-
- case SELECT:
- // This is used when converting multiset queries:
- //
- // select * from unnest(select multiset[deptno] from emps);
- //
- converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR,
- subQuery.logic, true, null);
- assert !converted.right;
- subQuery.expr = bb.register(converted.left, JoinRelType.LEFT);
- return;
-
- default:
- throw Util.newInternal("unexpected kind of subquery :" + subQuery.node);
- }
- }
-
- private RexNode translateIn(SubQuery subQuery, RelNode root,
- final RexNode rex) {
- switch (subQuery.logic) {
- case TRUE:
- return rexBuilder.makeLiteral(true);
-
- case UNKNOWN_AS_FALSE:
- assert rex instanceof RexRangeRef;
- final int fieldCount = rex.getType().getFieldCount();
- RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
- rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
-
- // Then append the IS NOT NULL(leftKeysForIn).
- //
- // RexRangeRef contains the following fields:
- // leftKeysForIn,
- // rightKeysForIn (the original subquery select list),
- // nullIndicator
- //
- // The first two lists contain the same number of fields.
- final int k = (fieldCount - 1) / 2;
- for (int i = 0; i < k; i++) {
- rexNode =
- rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- rexNode,
- rexBuilder.makeCall(
- SqlStdOperatorTable.IS_NOT_NULL,
- rexBuilder.makeFieldAccess(rex, i)));
- }
- return rexNode;
-
- case TRUE_FALSE_UNKNOWN:
- case UNKNOWN_AS_TRUE:
- // select e.deptno,
- // case
- // when ct.c = 0 then false
- // when dt.i is not null then true
- // when e.deptno is null then null
- // when ct.ck < ct.c then null
- // else false
- // end
- // from e
- // cross join (select count(*) as c, count(deptno) as ck from v) as ct
- // left join (select distinct deptno, true as i from v) as dt
- // on e.deptno = dt.deptno
- final Join join = (Join) root;
- final Project left = (Project) join.getLeft();
- final RelNode leftLeft = ((Join) left.getInput()).getLeft();
- final int leftLeftCount = leftLeft.getRowType().getFieldCount();
- final RelDataType nullableBooleanType =
- typeFactory.createTypeWithNullability(
- typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
- final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
- final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
- final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
- final RexNode iRef =
- rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
-
- final RexLiteral zero =
- rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
- final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
- final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
- final RexNode unknownLiteral =
- rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
-
- final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
- falseLiteral,
- rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
- trueLiteral);
- final JoinInfo joinInfo = join.analyzeCondition();
- for (int leftKey : joinInfo.leftKeys) {
- final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
- unknownLiteral);
- }
- args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
- unknownLiteral,
- falseLiteral);
-
- return rexBuilder.makeCall(
- nullableBooleanType,
- SqlStdOperatorTable.CASE,
- args.build());
-
- default:
- throw new AssertionError(subQuery.logic);
- }
- }
-
- private static boolean containsNullLiteral(SqlNodeList valueList) {
- for (SqlNode node : valueList.getList()) {
- if (node instanceof SqlLiteral) {
- SqlLiteral lit = (SqlLiteral) node;
- if (lit.getValue() == null) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Determines if a subquery is non-correlated and if so, converts it to a
- * constant.
- *
- * @param subQuery the call that references the subquery
- * @param bb blackboard used to convert the subquery
- * @param converted RelNode tree corresponding to the subquery
- * @param isExists true if the subquery is part of an EXISTS expression
- * @return if the subquery can be converted to a constant
- */
- private boolean convertNonCorrelatedSubQuery(
- SubQuery subQuery,
- Blackboard bb,
- RelNode converted,
- boolean isExists) {
- SqlCall call = (SqlBasicCall) subQuery.node;
- if (subqueryConverter.canConvertSubquery()
- && isSubQueryNonCorrelated(converted, bb)) {
- // First check if the subquery has already been converted
- // because it's a nested subquery. If so, don't re-evaluate
- // it again.
- RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
- if (constExpr == null) {
- constExpr =
- subqueryConverter.convertSubquery(
- call,
- this,
- isExists,
- isExplain);
- }
- if (constExpr != null) {
- subQuery.expr = constExpr;
- mapConvertedNonCorrSubqs.put(call, constExpr);
- return true;
- }
- }
- return false;
- }
-
- /**
- * Converts the RelNode tree for a select statement to a select that
- * produces a single value.
- *
- * @param query the query
- * @param plan the original RelNode tree corresponding to the statement
- * @return the converted RelNode tree
- */
- public RelNode convertToSingleValueSubq(
- SqlNode query,
- RelNode plan) {
- // Check whether query is guaranteed to produce a single value.
- if (query instanceof SqlSelect) {
- SqlSelect select = (SqlSelect) query;
- SqlNodeList selectList = select.getSelectList();
- SqlNodeList groupList = select.getGroup();
-
- if ((selectList.size() == 1)
- && ((groupList == null) || (groupList.size() == 0))) {
- SqlNode selectExpr = selectList.get(0);
- if (selectExpr instanceof SqlCall) {
- SqlCall selectExprCall = (SqlCall) selectExpr;
- if (Util.isSingleValue(selectExprCall)) {
- return plan;
- }
- }
-
- // If there is a limit with 0 or 1,
- // it is ensured to produce a single value
- if (select.getFetch() != null
- && select.getFetch() instanceof SqlNumericLiteral) {
- SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
- if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
- return plan;
- }
- }
- }
- } else if (query instanceof SqlCall) {
- // If the query is (values ...),
- // it is necessary to look into the operands to determine
- // whether SingleValueAgg is necessary
- SqlCall exprCall = (SqlCall) query;
- if (exprCall.getOperator()
- instanceof SqlValuesOperator
- && Util.isSingleValue(exprCall)) {
- return plan;
- }
- }
-
- // If not, project SingleValueAgg
- return RelOptUtil.createSingleValueAggRel(
- cluster,
- plan);
- }
-
- /**
- * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...".
- *
- * @param leftKeys LHS
- * @param valuesList RHS
- * @param isNotIn is this a NOT IN operator
- * @return converted expression
- */
- private RexNode convertInToOr(
- final Blackboard bb,
- final List<RexNode> leftKeys,
- SqlNodeList valuesList,
- boolean isNotIn) {
- final List<RexNode> comparisons = new ArrayList<>();
- for (SqlNode rightVals : valuesList) {
- RexNode rexComparison;
- if (leftKeys.size() == 1) {
- rexComparison =
- rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- leftKeys.get(0),
- rexBuilder.ensureType(leftKeys.get(0).getType(),
- bb.convertExpression(rightVals), true));
- } else {
- assert rightVals instanceof SqlCall;
- final SqlBasicCall call = (SqlBasicCall) rightVals;
- assert (call.getOperator() instanceof SqlRowOperator)
- && call.operandCount() == leftKeys.size();
- rexComparison =
- RexUtil.composeConjunction(
- rexBuilder,
- Iterables.transform(
- Pair.zip(leftKeys, call.getOperandList()),
- new Function<Pair<RexNode, SqlNode>, RexNode>() {
- public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- pair.left,
- rexBuilder.ensureType(pair.left.getType(),
- bb.convertExpression(pair.right), true));
- }
- }),
- false);
- }
- comparisons.add(rexComparison);
- }
-
- RexNode result =
- RexUtil.composeDisjunction(rexBuilder, comparisons, true);
- assert result != null;
-
- if (isNotIn) {
- result =
- rexBuilder.makeCall(
- SqlStdOperatorTable.NOT,
- result);
- }
-
- return result;
- }
-
- /**
- * Gets the list size threshold under which {@link #convertInToOr} is used.
- * Lists of this size or greater will instead be converted to use a join
- * against an inline table
- * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a
- * predicate. A threshold of 0 forces usage of an inline table in all cases; a
- * threshold of Integer.MAX_VALUE forces usage of OR in all cases
- *
- * @return threshold, default {@link #IN_SUBQUERY_THRESHOLD}
- */
- protected int getInSubqueryThreshold() {
- /* OVERRIDE POINT */
- return Integer.MAX_VALUE;
- }
-
- /**
- * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery
- * produces an indicator variable, and the result is a relational expression
- * which outer joins that indicator to the original query. After performing
- * the outer join, the condition will be TRUE if the EXISTS condition holds,
- * NULL otherwise.
- *
- * @param seek A query, for example 'select * from emp' or
- * 'values (1,2,3)' or '('Foo', 34)'.
- * @param subqueryType Whether sub-query is IN, EXISTS or scalar
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept an
- * approximation (say representing UNKNOWN as FALSE)
- * @param needsOuterJoin Whether an outer join is needed
- * @return join expression
- * @pre extraExpr == null || extraName != null
- */
- private Pair<RelNode, Boolean> convertExists(
- SqlNode seek,
- RelOptUtil.SubqueryType subqueryType,
- RelOptUtil.Logic logic,
- boolean needsOuterJoin,
- RelDataType targetDataType) {
- final SqlValidatorScope seekScope =
- (seek instanceof SqlSelect)
- ? validator.getSelectScope((SqlSelect) seek)
- : null;
- final Blackboard seekBb = createBlackboard(seekScope, null, false);
- RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
-
- return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic,
- needsOuterJoin);
- }
-
- private RelNode convertQueryOrInList(
- Blackboard bb,
- SqlNode seek,
- RelDataType targetRowType) {
- // NOTE: Once we start accepting single-row queries as row constructors,
- // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
- // Z)). The SQL standard resolves the ambiguity by saying that a lone
- // select should be interpreted as a table expression, not a row
- // expression. The semantic difference is that a table expression can
- // return multiple rows.
- if (seek instanceof SqlNodeList) {
- return convertRowValues(
- bb,
- seek,
- ((SqlNodeList) seek).getList(),
- false,
- targetRowType);
- } else {
- return convertQueryRecursive(seek, false, null).project();
- }
- }
-
- private RelNode convertRowValues(
- Blackboard bb,
- SqlNode rowList,
- Collection<SqlNode> rows,
- boolean allowLiteralsOnly,
- RelDataType targetRowType) {
- // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
- // literals into a single LogicalValues; this gives the optimizer a smaller
- // input tree. For everything else (computed expressions, row
- // subqueries), we union each row in as a projection on top of a
- // LogicalOneRow.
-
- final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
- ImmutableList.builder();
- final RelDataType rowType;
- if (targetRowType != null) {
- rowType = targetRowType;
- } else {
- rowType =
- SqlTypeUtil.promoteToRowType(
- typeFactory,
- validator.getValidatedNodeType(rowList),
- null);
- }
-
- final List<RelNode> unionInputs = new ArrayList<>();
- for (SqlNode node : rows) {
- SqlBasicCall call;
- if (isRowConstructor(node)) {
- call = (SqlBasicCall) node;
- ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
- for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- operand.e,
- bb,
- rowType,
- operand.i);
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- if ((rexLiteral == null) || !shouldCreateValuesRel) {
- // fallback to convertRowConstructor
- tuple = null;
- break;
- }
- tuple.add(rexLiteral);
- }
- if (tuple != null) {
- tupleList.add(tuple.build());
- continue;
- }
- } else {
- RexLiteral rexLiteral =
- convertLiteralInValuesList(
- node,
- bb,
- rowType,
- 0);
- if ((rexLiteral != null) && shouldCreateValuesRel) {
- tupleList.add(ImmutableList.of(rexLiteral));
- continue;
- } else {
- if ((rexLiteral == null) && allowLiteralsOnly) {
- return null;
- }
- }
-
- // convert "1" to "row(1)"
- call =
- (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
- SqlParserPos.ZERO,
- node);
- }
- unionInputs.add(convertRowConstructor(bb, call));
- }
- LogicalValues values =
- LogicalValues.create(cluster, rowType, tupleList.build());
- RelNode resultRel;
- if (unionInputs.isEmpty()) {
- resultRel = values;
- } else {
- if (!values.getTuples().isEmpty()) {
- unionInputs.add(values);
- }
- resultRel = LogicalUnion.create(unionInputs, true);
- }
- leaves.add(resultRel);
- return resultRel;
- }
-
- private RexLiteral convertLiteralInValuesList(
- SqlNode sqlNode,
- Blackboard bb,
- RelDataType rowType,
- int iField) {
- if (!(sqlNode instanceof SqlLiteral)) {
- return null;
- }
- RelDataTypeField field = rowType.getFieldList().get(iField);
- RelDataType type = field.getType();
- if (type.isStruct()) {
- // null literals for weird stuff like UDT's need
- // special handling during type flattening, so
- // don't use LogicalValues for those
- return null;
- }
-
- RexNode literalExpr =
- exprConverter.convertLiteral(
- bb,
- (SqlLiteral) sqlNode);
-
- if (!(literalExpr instanceof RexLiteral)) {
- assert literalExpr.isA(SqlKind.CAST);
- RexNode child = ((RexCall) literalExpr).getOperands().get(0);
- assert RexLiteral.isNullLiteral(child);
-
- // NOTE jvs 22-Nov-2006: we preserve type info
- // in LogicalValues digest, so it's OK to lose it here
- return (RexLiteral) child;
- }
-
- RexLiteral literal = (RexLiteral) literalExpr;
-
- Comparable value = literal.getValue();
-
- if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
- BigDecimal roundedValue =
- NumberUtil.rescaleBigDecimal(
- (BigDecimal) value,
- type.getScale());
- return rexBuilder.makeExactLiteral(
- roundedValue,
- type);
- }
-
- if ((value instanceof NlsString)
- && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
- // pad fixed character type
- NlsString unpadded = (NlsString) value;
- return rexBuilder.makeCharLiteral(
- new NlsString(
- Spaces.padRight(unpadded.getValue(), type.getPrecision()),
- unpadded.getCharsetName(),
- unpadded.getCollation()));
- }
- return literal;
- }
-
- private boolean isRowConstructor(SqlNode node) {
- if (!(node.getKind() == SqlKind.ROW)) {
- return false;
- }
- SqlCall call = (SqlCall) node;
- return call.getOperator().getName().equalsIgnoreCase("row");
- }
-
- /**
- * Builds a list of all <code>IN</code> or <code>EXISTS</code> operators
- * inside SQL parse tree. Does not traverse inside queries.
- *
- * @param bb blackboard
- * @param node the SQL parse tree
- * @param logic Whether the answer needs to be in full 3-valued logic (TRUE,
- * FALSE, UNKNOWN) will be required, or whether we can accept
- * an approximation (say representing UNKNOWN as FALSE)
- * @param registerOnlyScalarSubqueries if set to true and the parse tree
- * corresponds to a variation of a select
- * node, only register it if it's a scalar
- * subquery
- */
- private void findSubqueries(
- Blackboard bb,
- SqlNode node,
- RelOptUtil.Logic logic,
- boolean registerOnlyScalarSubqueries) {
- final SqlKind kind = node.getKind();
- switch (kind) {
- case EXISTS:
- case SELECT:
- case MULTISET_QUERY_CONSTRUCTOR:
- case MULTISET_VALUE_CONSTRUCTOR:
- case ARRAY_QUERY_CONSTRUCTOR:
- case CURSOR:
- case SCALAR_QUERY:
- if (!registerOnlyScalarSubqueries
- || (kind == SqlKind.SCALAR_QUERY)) {
- bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE);
- }
- return;
- case IN:
- if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) {
- logic = logic.negate();
- }
- break;
- case NOT:
- logic = logic.negate();
- break;
- }
- if (node instanceof SqlCall) {
- if (kind == SqlKind.OR
- || kind == SqlKind.NOT) {
- // It's always correct to outer join subquery with
- // containing query; however, when predicates involve Or
- // or NOT, outer join might be necessary.
- bb.subqueryNeedsOuterJoin = true;
- }
- for (SqlNode operand : ((SqlCall) node).getOperandList()) {
- if (operand != null) {
- // In the case of an IN expression, locate scalar
- // subqueries so we can convert them to constants
- findSubqueries(
- bb,
- operand,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
- } else if (node instanceof SqlNodeList) {
- for (SqlNode child : (SqlNodeList) node) {
- findSubqueries(
- bb,
- child,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubqueries);
- }
- }
-
- // Now that we've located any scalar subqueries inside the IN
- // expression, register the IN expression itself. We need to
- // register the scalar subqueries first so they can be converted
- // before the IN expression is converted.
- if (kind == SqlKind.IN) {
- if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
- && !validator.getValidatedNodeType(node).isNullable()) {
- logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE;
- }
- // TODO: This conversion is only valid in the WHERE clause
- if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE
- && !bb.subqueryNeedsOuterJoin) {
- logic = RelOptUtil.Logic.TRUE;
- }
- bb.registerSubquery(node, logic);
- }
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format.
- *
- * @param node Expression to translate
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node) {
- Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
- final ParameterScope scope =
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
- final Blackboard bb = createBlackboard(scope, null, false);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts an expression from {@link SqlNode} to {@link RexNode} format,
- * mapping identifier references to predefined expressions.
- *
- * @param node Expression to translate
- * @param nameToNodeMap map from String to {@link RexNode}; when an
- * {@link SqlIdentifier} is encountered, it is used as a
- * key and translated to the corresponding value from
- * this map
- * @return Converted expression
- */
- public RexNode convertExpression(
- SqlNode node,
- Map<String, RexNode> nameToNodeMap) {
- final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
- for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
- nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
- }
- final ParameterScope scope =
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
- final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
- return bb.convertExpression(node);
- }
-
- /**
- * Converts a non-standard expression.
- *
- * <p>This method is an extension-point that derived classes can override. If
- * this method returns a null result, the normal expression translation
- * process will proceed. The default implementation always returns null.
- *
- * @param node Expression
- * @param bb Blackboard
- * @return null to proceed with the usual expression translation process
- */
- protected RexNode convertExtendedExpression(
- SqlNode node,
- Blackboard bb) {
- return null;
- }
-
- private RexNode convertOver(Blackboard bb, SqlNode node) {
- SqlCall call = (SqlCall) node;
- SqlCall aggCall = call.operand(0);
- SqlNode windowOrRef = call.operand(1);
- final SqlWindow window =
- validator.resolveWindow(windowOrRef, bb.scope, true);
- // ROW_NUMBER() expects specific kind of framing.
- if (aggCall.getOperator() == SqlStdOperatorTable.ROW_NUMBER) {
- window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO));
- window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO));
- window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO));
- }
- final SqlNodeList partitionList = window.getPartitionList();
- final ImmutableList.Builder<RexNode> partitionKeys =
- ImmutableList.builder();
- for (SqlNode partition : partitionList) {
- partitionKeys.add(bb.convertExpression(partition));
- }
- RexNode lowerBound = bb.convertExpression(window.getLowerBound());
- RexNode upperBound = bb.convertExpression(window.getUpperBound());
- SqlNodeList orderList = window.getOrderList();
- if ((orderList.size() == 0) && !window.isRows()) {
- // A logical range requires an ORDER BY clause. Use the implicit
- // ordering of this relation. There must be one, otherwise it would
- // have failed validation.
- orderList = bb.scope.getOrderList();
- if (orderList == null) {
- throw new AssertionError(
- "Relation should have sort key for implicit ORDER BY");
- }
- }
- final ImmutableList.Builder<RexFieldCollation> orderKeys =
- ImmutableList.builder();
- final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
- for (SqlNode order : orderList) {
- flags.clear();
- RexNode e = bb.convertSortExpression(order, flags);
- orderKeys.add(new RexFieldCollation(e, flags));
- }
- try {
- Util.permAssert(bb.window == null, "already in window agg mode");
- bb.window = window;
- RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
- rexAgg =
- rexBuilder.ensureType(
- validator.getValidatedNodeType(call), rexAgg, false);
-
- // Walk over the tree and apply 'over' to all agg functions. This is
- // necessary because the returned expression is not necessarily a call
- // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
- final RexShuttle visitor =
- new HistogramShuttle(
- partitionKeys.build(), orderKeys.build(),
- RexWindowBound.create(window.getLowerBound(), lowerBound),
- RexWindowBound.create(window.getUpperBound(), upperBound),
- window);
- return rexAgg.accept(visitor);
- } finally {
- bb.window = null;
- }
- }
-
- /**
- * Converts a FROM clause into a relational expression.
- *
- * @param bb Scope within which to resolve identifiers
- * @param from FROM clause of a query. Examples include:
- *
- * <ul>
- * <li>a single table ("SALES.EMP"),
- * <li>an aliased table ("EMP AS E"),
- * <li>a list of tables ("EMP, DEPT"),
- * <li>an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO =
- * DEPT.DEPTNO"),
- * <li>a VALUES clause ("VALUES ('Fred', 20)"),
- * <li>a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"),
- * <li>or any combination of the above.
- * </ul>
- */
- protected void convertFrom(
- Blackboard bb,
- SqlNode from) {
- final SqlCall call;
- final SqlNode[] operands;
- switch (from.getKind()) {
- case AS:
- convertFrom(bb, ((SqlCall) from).operand(0));
- return;
-
- case WITH_ITEM:
- convertFrom(bb, ((SqlWithItem) from).query);
- return;
-
- case WITH:
- convertFrom(bb, ((SqlWith) from).body);
- return;
-
- case TABLESAMPLE:
- operands = ((SqlBasicCall) from).getOperands();
- SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
- if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
- String sampleName =
- ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
- .getName();
- datasetStack.push(sampleName);
- convertFrom(bb, operands[0]);
- datasetStack.pop();
- } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
- SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
- (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
- convertFrom(bb, operands[0]);
- RelOptSamplingParameters params =
- new RelOptSamplingParameters(
- tableSampleSpec.isBernoulli(),
- tableSampleSpec.getSamplePercentage(),
- tableSampleSpec.isRepeatable(),
- tableSampleSpec.getRepeatableSeed());
- bb.setRoot(new Sample(cluster, bb.root, params), false);
- } else {
- throw Util.newInternal(
- "unknown TABLESAMPLE type: " + sampleSpec);
- }
- return;
-
- case IDENTIFIER:
- final SqlValidatorNamespace fromNamespace =
- validator.getNamespace(from).resolve();
- if (fromNamespace.getNode() != null) {
- convertFrom(bb, fromNamespace.getNode());
- return;
- }
- final String datasetName =
- datasetStack.isEmpty() ? null : datasetStack.peek();
- boolean[] usedDataset = {false};
- RelOptTable table =
- SqlValidatorUtil.getRelOptTable(
- fromNamespace,
- catalogReader,
- datasetName,
- usedDataset);
- final RelNode tableRel;
- if (shouldConvertTableAccess) {
- tableRel = toRel(table);
- } else {
- tableRel = LogicalTableScan.create(cluster, table);
- }
- bb.setRoot(tableRel, true);
- if (usedDataset[0]) {
- bb.setDataset(datasetName);
- }
- return;
-
- case JOIN:
- final SqlJoin join = (SqlJoin) from;
- final SqlValidatorScope scope = validator.getJoinScope(from);
- final Blackboard fromBlackboard = createBlackboard(scope, null, false);
- SqlNode left = join.getLeft();
- SqlNode right = join.getRight();
- final boolean isNatural = join.isNatural();
- final JoinType joinType = join.getJoinType();
- final SqlValidatorScope leftScope =
- Util.first(validator.getJoinScope(left),
- ((DelegatingScope) bb.scope).getParent());
- final Blackboard leftBlackboard =
- createBlackboard(leftScope, null, false);
- final SqlValidatorScope rightScope =
- Util.first(validator.getJoinScope(right),
- ((DelegatingScope) bb.scope).getParent());
- final Blackboard rightBlackboard =
- createBlackboard(rightScope, null, false);
- convertFrom(leftBlackboard, left);
- RelNode leftRel = leftBlackboard.root;
- convertFrom(rightBlackboard, right);
- RelNode rightRel = rightBlackboard.root;
- JoinRelType convertedJoinType = convertJoinType(joinType);
- RexNode conditionExp;
- final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
- final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
- if (isNatural) {
- final RelDataType leftRowType = leftNamespace.getRowType();
- final RelDataType rightRowType = rightNamespace.getRowType();
- final List<String> columnList =
- SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType,
- rightRowType);
- conditionExp = convertUsing(leftNamespace, rightNamespace,
- columnList);
- } else {
- conditionExp =
- convertJoinCondition(
- fromBlackboard,
- leftNamespace,
- rightNamespace,
- join.getCondition(),
- join.getConditionType(),
- leftRel,
- rightRel);
- }
-
- final RelNode joinRel =
- createJoin(
- fromBlackboard,
- leftRel,
- rightRel,
- conditionExp,
- convertedJoinType);
- bb.setRoot(joinRel, false);
- return;
-
- case SELECT:
- case INTERSECT:
- case EXCEPT:
- case UNION:
- final RelNode rel = convertQueryRecursive(from, false, null).project();
- bb.setRoot(rel, true);
- return;
-
- case VALUES:
- convertValuesImpl(bb, (SqlCall) from, null);
- return;
-
- case UNNEST:
- call = (SqlCall) from;
- final SqlNode node = call.operand(0);
- final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator();
- replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- final RelNode childRel =
- RelOptUtil.createProject(
- (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
- Collections.singletonList(bb.convertExpression(node)),
- Collections.singletonList(validator.deriveAlias(node, 0)),
- true);
-
- Uncollect uncollect =
- new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
- childRel, operator.withOrdinality);
- bb.setRoot(uncollect, true);
- return;
-
- case COLLECTION_TABLE:
- call = (SqlCall) from;
-
- // Dig out real call; TABLE() wrapper is just syntactic.
- assert call.getOperandList().size() == 1;
- final SqlCall call2 = call.operand(0);
- convertCollectionTable(bb, call2);
- return;
-
- default:
- throw Util.newInternal("not a join operator " + from);
- }
- }
-
- protected void convertCollectionTable(
- Blackboard bb,
- SqlCall call) {
- final SqlOperator operator = call.getOperator();
- if (operator == SqlStdOperatorTable.TABLESAMPLE) {
- final String sampleName =
- SqlLiteral.stringValue(call.operand(0));
- datasetStack.push(sampleName);
- SqlCall cursorCall = call.operand(1);
- SqlNode query = cursorCall.operand(0);
- RelNode converted = convertQuery(query, false, false).rel;
- bb.setRoot(converted, false);
- datasetStack.pop();
- return;
- }
- replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
- // Expand table macro if possible. It's more efficient than
- // LogicalTableFunctionScan.
- final SqlCallBinding callBinding =
- new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
- if (operator instanceof SqlUserDefinedTableMacro) {
- final SqlUserDefinedTableMacro udf =
- (SqlUserDefinedTableMacro) operator;
- final TranslatableTable table =
- udf.getTable(typeFactory, callBinding.operands());
- final RelDataType rowType = table.getRowType(typeFactory);
- RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
- RelNode converted = toRel(relOptTable);
- bb.setRoot(converted, true);
- return;
- }
-
- Type elementType;
- if (operator instanceof SqlUserDefinedTableFunction) {
- SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
- elementType = udtf.getElementType(typeFactory, callBinding.operands());
- } else {
- elementType = null;
- }
-
- RexNode rexCall = bb.convertExpression(call);
- final List<RelNode> inputs = bb.retrieveCursors();
- Set<RelColumnMapping> columnMappings =
- getColumnMappings(operator);
- LogicalTableFunctionScan callRel =
- LogicalTableFunctionScan.create(
- cluster,
- inputs,
- rexCall,
- elementType,
- validator.getValidatedNodeType(call),
- columnMappings);
- bb.setRoot(callRel, true);
- afterTableFunction(bb, call, callRel);
- }
-
- protected void afterTableFunction(
- SqlToRelConverter.Blackboard bb,
- SqlCall call,
- LogicalTableFunctionScan callRel) {
- }
-
- private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
- SqlReturnTypeInference rti = op.getReturnTypeInference();
- if (rti == null) {
- return null;
- }
- if (rti instanceof TableFunctionReturnTypeInference) {
- TableFunctionReturnTypeInference tfrti =
- (TableFunctionReturnTypeInference) rti;
- return tfrti.getColumnMappings();
- } else {
- return null;
- }
- }
-
- protected RelNode createJoin(
- Blackboard bb,
- RelNode leftRel,
- RelNode rightRel,
- RexNode joinCond,
- JoinRelType joinType) {
- assert joinCond != null;
-
- final CorrelationUse p = getCorrelationUse(bb, rightRel);
- if (p != null) {
- LogicalCorrelate corr = LogicalCorrelate.create(leftRel, p.r,
- p.id, p.requiredColumns, SemiJoinType.of(joinType));
- if (!joinCond.isAlwaysTrue()) {
- return RelOptUtil.createFilter(corr, joinCond);
- }
- return corr;
- }
-
- final Join originalJoin =
- (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
- joinCond, ImmutableSet.<CorrelationId>of(), joinType, false);
-
- return RelOptUtil.pushDownJoinConditions(originalJoin);
- }
-
- private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
- final Set<CorrelationId> correlatedVariables =
- RelOptUtil.getVariablesUsed(r0);
- if (correlatedVariables.isEmpty()) {
- return null;
- }
- final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
- final List<CorrelationId> correlNames = Lists.newArrayList();
-
- // All correlations must refer the same namespace since correlation
- // produces exactly one correlation source.
- // The same source might be referenced by different variables since
- // DeferredLookups are not de-duplicated at create time.
- SqlValidatorNamespace prevNs = null;
-
- for (CorrelationId correlName : correlatedVariables) {
- DeferredLookup lookup =
- mapCorrelToDeferred.get(correlName);
- RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
- String originalRelName = lookup.getOriginalRelName();
- String originalFieldName = fieldAccess.getField().getName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- ImmutableList.of(originalRelName),
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- int childNamespaceIndex = nsIndexes[0];
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
- boolean correlInCurrentScope = ancestorScope == bb.scope;
-
- if (!correlInCurrentScope) {
- continue;
- }
-
- if (prevNs == null) {
- prevNs = foundNs;
- } else {
- assert prevNs == foundNs : "All correlation variables should resolve"
- + " to the same namespace."
- + " Prev ns=" + prevNs
- + ", new ns=" + foundNs;
- }
-
- int namespaceOffset = 0;
- if (childNamespaceIndex > 0) {
- // If not the first child, need to figure out the width
- // of output types from all the preceding namespaces
- assert ancestorScope instanceof ListScope;
- List<SqlValidatorNamespace> children =
- ((ListScope) ancestorScope).getChildren();
-
- for (int i = 0; i < childNamespaceIndex; i++) {
- SqlValidatorNamespace child = children.get(i);
- namespaceOffset +=
- child.getRowType().getFieldCount();
- }
- }
-
- RelDataTypeField field =
- catalogReader.field(foundNs.getRowType(), originalFieldName);
- int pos = namespaceOffset + field.getIndex();
-
- assert field.getType()
- == lookup.getFieldAccess(correlName).getField().getType();
-
- assert pos != -1;
-
- if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
- // bb.root is an aggregate and only projects group by
- // keys.
- Map<Integer, Integer> exprProjection =
- bb.mapRootRelToFieldProjection.get(bb.root);
-
- // subquery can reference group by keys projected from
- // the root of the outer relation.
- if (exprProjection.containsKey(pos)) {
- pos = exprProjection.get(pos);
- } else {
- // correl not grouped
- throw new AssertionError("Identifier '" + originalRelName + "."
- + originalFieldName + "' is not a group expr");
- }
- }
-
- requiredColumns.set(pos);
- correlNames.add(correlName);
- }
-
- if (correlNames.isEmpty()) {
- // None of the correlating variables originated in this scope.
- return null;
- }
-
- RelNode r = r0;
- if (correlNames.size() > 1) {
- // The same table was referenced more than once.
- // So we deduplicate
- r = DeduplicateCorrelateVariables.go(rexBuilder, correlNames.get(0),
- Util.skip(correlNames), r0);
- }
- return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r);
- }
-
- /**
- * Determines whether a subquery is non-correlated. Note that a
- * non-correlated subquery can contain correlated references, provided those
- * references do not reference select statements that are parents of the
- * subquery.
- *
- * @param subq the subquery
- * @param bb blackboard used while converting the subquery, i.e., the
- * blackboard of the parent query of this subquery
- * @return true if the subquery is non-correlated.
- */
- private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
- Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
- for (CorrelationId correlName : correlatedVariables) {
- DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
- String originalRelName = lookup.getOriginalRelName();
-
- int[] nsIndexes = {-1};
- final SqlValidatorScope[] ancestorScopes = {null};
- SqlValidatorNamespace foundNs =
- lookup.bb.scope.resolve(
- ImmutableList.of(originalRelName),
- ancestorScopes,
- nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
-
- // If the correlated reference is in a scope that's "above" the
- // subquery, then this is a correlated subquery.
- SqlValidatorScope parentScope = bb.scope;
- do {
- if (ancestorScope == parentScope) {
- return false;
- }
- if (parentScope instanceof DelegatingScope) {
- parentScope = ((DelegatingScope) parentScope).getParent();
- } else {
- break;
- }
- } while (parentScope != null);
- }
- return true;
- }
-
- /**
- * Returns a list of fields to be prefixed to each relational expression.
- *
- * @return List of system fields
- */
- protected List<RelDataTypeField> getSystemFields() {
- return Collections.emptyList();
- }
-
- private RexNode convertJoinCondition(Blackboard bb,
- SqlValidatorNamespace leftNamespace,
- SqlValidatorNamespace rightNamespace,
- SqlNode condition,
- JoinConditionType conditionType,
- RelNode leftRel,
- RelNode rightRel) {
- if (condition == null) {
- return rexBuilder.makeLiteral(true);
- }
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
- switch (conditionType) {
- case ON:
- bb.setRoot(ImmutableList.of(leftRel, rightRel));
- return bb.convertExpression(condition);
- case USING:
- final SqlNodeList list = (SqlNodeList) condition;
- final List<String> nameList = new ArrayList<>();
- for (SqlNode columnName : list) {
- final SqlIdentifier id = (SqlIdentifier) columnName;
- String name = id.getSimple();
- nameList.add(name);
- }
- return convertUsing(leftNamespace, rightNamespace, nameList);
- default:
- throw Util.unexpected(conditionType);
- }
- }
-
- /**
- * Returns an expression for matching columns of a USING clause or inferred
- * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y =
- * b.y". Returns null if the column list is empty.
- *
- * @param leftNamespace Namespace of left input to join
- * @param rightNamespace Namespace of right input to join
- * @param nameList List of column names to join on
- * @return Expression to match columns from name list, or true if name list
- * is empty
- */
- private RexNode convertUsing(SqlValidatorNamespace leftNamespace,
- SqlValidatorNamespace rightNamespace,
- List<String> nameList) {
- final List<RexNode> list = Lists.newArrayList();
- for (String name : nameList) {
- List<RexNode> operands = new ArrayList<>();
- int offset = 0;
- for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace,
- rightNamespace)) {
- final RelDataType rowType = n.getRowType();
- final RelDataTypeField field = catalogReader.field(rowType, name);
- operands.add(
- rexBuilder.makeInputRef(field.getType(),
- offset + field.getIndex()));
- offset += rowType.getFieldList().size();
- }
- list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands));
- }
- return RexUtil.composeConjuncti
<TRUNCATED>
[10/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
deleted file mode 100644
index 698a978..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ /dev/null
@@ -1,552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-import static org.apache.hadoop.util.StringUtils.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.RawResource;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.cmd.ShellCmdOutput;
-import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.job.tools.OptionsHelper;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("static-access")
-public abstract class AbstractHadoopJob extends Configured implements Tool {
- protected static final Logger logger = LoggerFactory.getLogger(AbstractHadoopJob.class);
-
- protected static final Option OPTION_JOB_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Job name. For exmaple, Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create("jobname");
- protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create("cubename");
- protected static final Option OPTION_II_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create("iiname");
- protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Cube segment name)").create("segmentname");
- protected static final Option OPTION_TABLE_NAME = OptionBuilder.withArgName("name").hasArg().isRequired(true).withDescription("Hive table name.").create("tablename");
- protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Input path").create("input");
- protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName("inputformat").hasArg().isRequired(false).withDescription("Input format").create("inputformat");
- protected static final Option OPTION_INPUT_DELIM = OptionBuilder.withArgName("inputdelim").hasArg().isRequired(false).withDescription("Input delimeter").create("inputdelim");
- protected static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Output path").create("output");
- protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName("level").hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create("level");
- protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Partition file path.").create("input");
- protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName("htable name").hasArg().isRequired(true).withDescription("HTable name").create("htablename");
-
- protected String name;
- protected boolean isAsync = false;
- protected OptionsHelper optionsHelper = new OptionsHelper();
-
- protected Job job;
-
- protected void parseOptions(Options options, String[] args) throws ParseException {
- optionsHelper.parseOptions(options, args);
- }
-
- public void printUsage(Options options) {
- optionsHelper.printUsage(getClass().getSimpleName(), options);
- }
-
- public Option[] getOptions() {
- return optionsHelper.getOptions();
- }
-
- public String getOptionsAsString() {
- return optionsHelper.getOptionsAsString();
- }
-
- protected String getOptionValue(Option option) {
- return optionsHelper.getOptionValue(option);
- }
-
- protected boolean hasOption(Option option) {
- return optionsHelper.hasOption(option);
- }
-
- protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException {
- int retVal = 0;
- long start = System.nanoTime();
- if (isAsync) {
- job.submit();
- } else {
- job.waitForCompletion(true);
- retVal = job.isSuccessful() ? 0 : 1;
- logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L));
- }
- return retVal;
- }
-
- private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec.jar|[^,]*hive-metastore.jar|[^,]*hive-hcatalog-core[0-9.-]*jar";
-
- String filterKylinHiveDependency(String kylinHiveDependency) {
- if (StringUtils.isBlank(kylinHiveDependency))
- return "";
-
- StringBuilder jarList = new StringBuilder();
-
- Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS);
- Matcher matcher = hivePattern.matcher(kylinHiveDependency);
-
- while (matcher.find()) {
- if (jarList.length() > 0)
- jarList.append(",");
- jarList.append(matcher.group());
- }
-
- return jarList.toString();
- }
-
- private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
-
- protected void setJobClasspath(Job job) {
- String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath();
- File jarFile = new File(jarPath);
- if (jarFile.exists()) {
- job.setJar(jarPath);
- logger.info("append job jar: " + jarPath);
- } else {
- job.setJarByClass(this.getClass());
- }
-
- String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
- String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
- logger.info("append kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
-
- Configuration jobConf = job.getConfiguration();
- String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null || classpath.length() == 0) {
- logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
- classpath = getDefaultMapRedClasspath();
- classpath = classpath.replace(":", ","); // yarn classpath is comma separated
- logger.info("The default mapred classpath is: " + classpath);
- }
-
- if (kylinHBaseDependency != null) {
- // yarn classpath is comma separated
- kylinHBaseDependency = kylinHBaseDependency.replace(":", ",");
- classpath = classpath + "," + kylinHBaseDependency;
- }
-
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
- logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
-
- /*
- * set extra dependencies as tmpjars & tmpfiles if configured
- */
- StringBuilder kylinDependency = new StringBuilder();
-
- // for hive dependencies
- if (kylinHiveDependency != null) {
- // yarn classpath is comma separated
- kylinHiveDependency = kylinHiveDependency.replace(":", ",");
-
- logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency);
- String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
- logger.info("Hive Dependencies After Filtered: " + filteredHive);
-
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(filteredHive);
- }
-
- // for KylinJobMRLibDir
- String mrLibDir = KylinConfig.getInstanceFromEnv().getKylinJobMRLibDir();
- if (!StringUtils.isBlank(mrLibDir)) {
- File dirFileMRLIB = new File(mrLibDir);
- if (dirFileMRLIB.exists()) {
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(mrLibDir);
- } else {
- logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!");
- }
- }
-
- setJobTmpJarsAndFiles(job, kylinDependency.toString());
- }
-
- private void setJobTmpJarsAndFiles(Job job, String kylinDependency) {
- if (StringUtils.isBlank(kylinDependency))
- return;
-
- String[] fNameList = kylinDependency.split(",");
-
- try {
- Configuration jobConf = job.getConfiguration();
- FileSystem fs = FileSystem.getLocal(jobConf);
-
- StringBuilder jarList = new StringBuilder();
- StringBuilder fileList = new StringBuilder();
-
- for (String fileName : fNameList) {
- Path p = new Path(fileName);
- if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, fileName);
- continue;
- }
-
- StringBuilder list = (p.getName().endsWith(".jar")) ? jarList : fileList;
- if (list.length() > 0)
- list.append(",");
- list.append(fs.getFileStatus(p).getPath().toString());
- }
-
- appendTmpFiles(fileList.toString(), jobConf);
- appendTmpJars(jarList.toString(), jobConf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private void appendTmpDir(Job job, String tmpDir) {
- if (StringUtils.isBlank(tmpDir))
- return;
-
- try {
- Configuration jobConf = job.getConfiguration();
- FileSystem fs = FileSystem.getLocal(jobConf);
- FileStatus[] fList = fs.listStatus(new Path(tmpDir));
-
- StringBuilder jarList = new StringBuilder();
- StringBuilder fileList = new StringBuilder();
-
- for (FileStatus file : fList) {
- Path p = file.getPath();
- if (fs.getFileStatus(p).isDirectory()) {
- appendTmpDir(job, p.toString());
- continue;
- }
-
- StringBuilder list = (p.getName().endsWith(".jar")) ? jarList : fileList;
- if (list.length() > 0)
- list.append(",");
- list.append(fs.getFileStatus(p).getPath().toString());
- }
-
- appendTmpFiles(fileList.toString(), jobConf);
- appendTmpJars(jarList.toString(), jobConf);
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private void appendTmpJars(String jarList, Configuration conf) {
- if (StringUtils.isBlank(jarList))
- return;
-
- String tmpJars = conf.get("tmpjars", null);
- if (tmpJars == null) {
- tmpJars = jarList;
- } else {
- tmpJars += "," + jarList;
- }
- conf.set("tmpjars", tmpJars);
- logger.info("Job 'tmpjars' updated -- " + tmpJars);
- }
-
- private void appendTmpFiles(String fileList, Configuration conf) {
- if (StringUtils.isBlank(fileList))
- return;
-
- String tmpFiles = conf.get("tmpfiles", null);
- if (tmpFiles == null) {
- tmpFiles = fileList;
- } else {
- tmpFiles += "," + fileList;
- }
- conf.set("tmpfiles", tmpFiles);
- logger.info("Job 'tmpfiles' updated -- " + tmpFiles);
- }
-
- private String getDefaultMapRedClasspath() {
-
- String classpath = "";
- try {
- CliCommandExecutor executor = KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
- ShellCmdOutput output = new ShellCmdOutput();
- executor.execute("mapred classpath", output);
-
- classpath = output.getOutput().trim();
- } catch (IOException e) {
- logger.error("Failed to run: 'mapred classpath'.", e);
- }
-
- return classpath;
- }
-
- public void addInputDirs(String input, Job job) throws IOException {
- for (String inp : StringSplitter.split(input, ",")) {
- inp = inp.trim();
- if (inp.endsWith("/*")) {
- inp = inp.substring(0, inp.length() - 2);
- FileSystem fs = FileSystem.get(job.getConfiguration());
- Path path = new Path(inp);
- FileStatus[] fileStatuses = fs.listStatus(path);
- boolean hasDir = false;
- for (FileStatus stat : fileStatuses) {
- if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) {
- hasDir = true;
- addInputDirs(stat.getPath().toString(), job);
- }
- }
- if (fileStatuses.length > 0 && !hasDir) {
- addInputDirs(path.toString(), job);
- }
- } else {
- logger.debug("Add input " + inp);
- FileInputFormat.addInputPath(job, new Path(inp));
- }
- }
- }
-
- protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException {
- File tmp = File.createTempFile("kylin_job_meta", "");
- FileUtils.forceDelete(tmp);
-
- File metaDir = new File(tmp, "meta");
- metaDir.mkdirs();
-
- // write kylin.properties
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- File kylinPropsFile = new File(metaDir, "kylin.properties");
- kylinConfig.writeProperties(kylinPropsFile);
-
- // write cube / model_desc / cube_desc / dict / table
- ArrayList<String> dumpList = new ArrayList<String>();
- dumpList.add(cube.getResourcePath());
- dumpList.add(cube.getDescriptor().getModel().getResourcePath());
- dumpList.add(cube.getDescriptor().getResourcePath());
- for (String tableName : cube.getDescriptor().getModel().getAllTables()) {
- TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName);
- dumpList.add(table.getResourcePath());
- }
-
- for (CubeSegment segment : cube.getSegments()) {
- dumpList.addAll(segment.getDictionaryPaths());
- }
-
- dumpResources(kylinConfig, metaDir, dumpList);
- addToHadoopDistCache(conf, metaDir);
- }
-
- protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
- File tmp = File.createTempFile("kylin_job_meta", "");
- FileUtils.forceDelete(tmp);
-
- File metaDir = new File(tmp, "meta");
- metaDir.mkdirs();
-
- // write kylin.properties
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- File kylinPropsFile = new File(metaDir, "kylin.properties");
- kylinConfig.writeProperties(kylinPropsFile);
-
- // write II / model_desc / II_desc / dict / table
- ArrayList<String> dumpList = new ArrayList<String>();
- dumpList.add(ii.getResourcePath());
- dumpList.add(ii.getDescriptor().getModel().getResourcePath());
- dumpList.add(ii.getDescriptor().getResourcePath());
-
- for (String tableName : ii.getDescriptor().getModel().getAllTables()) {
- TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName);
- dumpList.add(table.getResourcePath());
- }
-
- for (IISegment segment : ii.getSegments()) {
- dumpList.addAll(segment.getDictionaryPaths());
- }
-
- dumpResources(kylinConfig, metaDir, dumpList);
- addToHadoopDistCache(conf, metaDir);
- }
-
- private void addToHadoopDistCache(Configuration conf, File metaDir) {
- // hadoop distributed cache
- String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
- if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..."
- hdfsMetaDir = "file://" + hdfsMetaDir;
- else
- hdfsMetaDir = "file:///" + hdfsMetaDir;
- logger.info("HDFS meta dir is: " + hdfsMetaDir);
-
- appendTmpFiles(hdfsMetaDir, conf);
- }
-
- private void dumpResources(KylinConfig kylinConfig, File metaDir, ArrayList<String> dumpList) throws IOException {
- ResourceStore from = ResourceStore.getStore(kylinConfig);
- KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
- ResourceStore to = ResourceStore.getStore(localConfig);
- for (String path : dumpList) {
- RawResource res = from.getResource(path);
- if (res == null)
- throw new IllegalStateException("No resource found at -- " + path);
- to.putResource(path, res.inputStream, res.timestamp);
- res.inputStream.close();
- }
- }
-
- protected void deletePath(Configuration conf, Path path) throws IOException {
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- }
-
- protected double getTotalMapInputMB() throws ClassNotFoundException, IOException, InterruptedException, JobException {
- if (job == null) {
- throw new JobException("Job is null");
- }
-
- long mapInputBytes = 0;
- InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
- for (InputSplit split : input.getSplits(job)) {
- mapInputBytes += split.getLength();
- }
- if (mapInputBytes == 0) {
- throw new IllegalArgumentException("Map input splits are 0 bytes, something is wrong!");
- }
- double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
- return totalMapInputMB;
- }
-
- protected int getMapInputSplitCount() throws ClassNotFoundException, JobException, IOException, InterruptedException {
- if (job == null) {
- throw new JobException("Job is null");
- }
- InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
- return input.getSplits(job).size();
- }
-
- public static KylinConfig loadKylinPropsAndMetadata(Configuration conf) throws IOException {
- File metaDir = new File("meta");
- System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath());
- logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath());
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- kylinConfig.setMetadataUrl(metaDir.getAbsolutePath());
- return kylinConfig;
- }
-
- protected void cleanupTempConfFile(Configuration conf) {
- String tempMetaFileString = conf.get("tmpfiles");
- logger.info("tempMetaFileString is : " + tempMetaFileString);
- if (tempMetaFileString != null) {
- if (tempMetaFileString.startsWith("file://")) {
- tempMetaFileString = tempMetaFileString.substring("file://".length());
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- try {
- FileUtils.forceDelete(tempMetaFile.getParentFile());
-
- } catch (IOException e) {
- logger.warn("error when deleting " + tempMetaFile, e);
- }
- } else {
- logger.info("" + tempMetaFileString + " does not exist");
- }
- } else {
- logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
- }
- }
- }
-
- public void kill() throws JobException {
- if (job != null) {
- try {
- job.killJob();
- } catch (IOException e) {
- throw new JobException(e);
- }
- }
- }
-
- public Map<String, String> getInfo() throws JobException {
- if (job != null) {
- Map<String, String> status = new HashMap<String, String>();
- if (null != job.getJobID()) {
- status.put(JobInstance.MR_JOB_ID, job.getJobID().toString());
- }
- if (null != job.getTrackingURL()) {
- status.put(JobInstance.YARN_APP_URL, job.getTrackingURL().toString());
- }
-
- return status;
- } else {
- throw new JobException("Job is null");
- }
- }
-
- public Counters getCounters() throws JobException {
- if (job != null) {
- try {
- return job.getCounters();
- } catch (IOException e) {
- throw new JobException(e);
- }
- } else {
- throw new JobException("Job is null");
- }
- }
-
- public void setAsync(boolean isAsync) {
- this.isAsync = isAsync;
- }
-
- public Job getJob() {
- return this.job;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
deleted file mode 100644
index 787181c..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cardinality;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.kv.RowConstants;
-
-/**
- * @author Jack
- *
- */
-public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWritable, BytesWritable> {
-
- private Map<Integer, HyperLogLogPlusCounter> hllcMap = new HashMap<Integer, HyperLogLogPlusCounter>();
- public static final String DEFAULT_DELIM = ",";
-
- private int counter = 0;
-
- private HCatSchema schema = null;
- private int columnSize = 0;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- schema = HCatInputFormat.getTableSchema(context.getConfiguration());
- columnSize = schema.getFields().size();
- }
-
- @Override
- public void map(T key, HCatRecord value, Context context) throws IOException, InterruptedException {
-
- HCatFieldSchema field;
- Object fieldValue;
- for (int m = 0; m < columnSize; m++) {
- field = schema.get(m);
- fieldValue = value.get(field.getName(), schema);
- if (fieldValue == null)
- fieldValue = "NULL";
-
- if (counter < 5 && m < 10) {
- System.out.println("Get row " + counter + " column '" + field.getName() + "' value: " + fieldValue);
- }
-
- if (fieldValue != null)
- getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
- }
-
- counter++;
- }
-
- private HyperLogLogPlusCounter getHllc(Integer key) {
- if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounter());
- }
- return hllcMap.get(key);
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- Iterator<Integer> it = hllcMap.keySet().iterator();
- while (it.hasNext()) {
- int key = it.next();
- HyperLogLogPlusCounter hllc = hllcMap.get(key);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit()));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
deleted file mode 100644
index ab4285a..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cardinality;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.cube.kv.RowConstants;
-
-/**
- * @author Jack
- *
- */
-public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWritable, IntWritable, LongWritable> {
-
- public static final int ONE = 1;
- private Map<Integer, HyperLogLogPlusCounter> hllcMap = new HashMap<Integer, HyperLogLogPlusCounter>();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- }
-
- @Override
- public void reduce(IntWritable key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
- int skey = key.get();
- for (BytesWritable v : values) {
- ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
- hll.readRegisters(buffer);
- getHllc(skey).merge(hll);
- hll.clear();
- }
- }
-
- private HyperLogLogPlusCounter getHllc(Integer key) {
- if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounter());
- }
- return hllcMap.get(key);
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- List<Integer> keys = new ArrayList<Integer>();
- Iterator<Integer> it = hllcMap.keySet().iterator();
- while (it.hasNext()) {
- keys.add(it.next());
- }
- Collections.sort(keys);
- it = keys.iterator();
- while (it.hasNext()) {
- int key = it.next();
- HyperLogLogPlusCounter hllc = hllcMap.get(key);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
- // context.write(new Text("ErrorRate_" + key), new
- // LongWritable((long)hllc.getErrorRate()));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
deleted file mode 100644
index f27d074..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cardinality;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-
-/**
- * This hadoop job will scan all rows of the hive table and then calculate the cardinality on each column.
- * @author shaoshi
- *
- */
-public class HiveColumnCardinalityJob extends AbstractHadoopJob {
- public static final String JOB_TITLE = "Kylin Hive Column Cardinality Job";
-
- @SuppressWarnings("static-access")
- protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table");
-
- public static final String OUTPUT_PATH = BatchConstants.CFG_KYLIN_HDFS_TEMP_DIR + "cardinality";
-
- public HiveColumnCardinalityJob() {
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- Options options = new Options();
-
- try {
- options.addOption(OPTION_TABLE);
- options.addOption(OPTION_OUTPUT_PATH);
-
- parseOptions(options, args);
-
- // start job
- String jobName = JOB_TITLE + getOptionsAsString();
- System.out.println("Starting: " + jobName);
- Configuration conf = getConf();
-
- JobEngineConfig jobEngineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
- conf.addResource(jobEngineConfig.getHadoopJobConfFilePath(null));
-
- job = Job.getInstance(conf, jobName);
-
- setJobClasspath(job);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- FileOutputFormat.setOutputPath(job, output);
- job.getConfiguration().set("dfs.block.size", "67108864");
-
- // Mapper
- String table = getOptionValue(OPTION_TABLE);
- String[] dbTableNames = HadoopUtil.parseHiveTableName(table);
- HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);
-
- job.setInputFormatClass(HCatInputFormat.class);
- job.setMapperClass(ColumnCardinalityMapper.class);
- job.setMapOutputKeyClass(IntWritable.class);
- job.setMapOutputValueClass(BytesWritable.class);
-
- // Reducer - only one
- job.setReducerClass(ColumnCardinalityReducer.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- job.setOutputKeyClass(IntWritable.class);
- job.setOutputValueClass(LongWritable.class);
- job.setNumReduceTasks(1);
-
- this.deletePath(job.getConfiguration(), output);
-
- System.out.println("Going to submit HiveColumnCardinalityJob for table '" + table + "'");
- int result = waitForCompletion(job);
-
- return result;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
deleted file mode 100644
index 7bd3814..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cardinality;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This job will update save the cardinality result into Kylin table metadata store.
- *
- * @author shaoshi
- */
-public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
- public static final String JOB_TITLE = "Kylin Hive Column Cardinality Update Job";
-
- @SuppressWarnings("static-access")
- protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table");
-
- private static final Logger logger = LoggerFactory.getLogger(HiveColumnCardinalityUpdateJob.class);
- private String table;
-
- public HiveColumnCardinalityUpdateJob() {
-
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- Options options = new Options();
-
- try {
- options.addOption(OPTION_TABLE);
- options.addOption(OPTION_OUTPUT_PATH);
-
- parseOptions(options, args);
-
- this.table = getOptionValue(OPTION_TABLE).toUpperCase();
- // start job
- String jobName = JOB_TITLE + getOptionsAsString();
- logger.info("Starting: " + jobName);
- Configuration conf = getConf();
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- updateKylinTableExd(table.toUpperCase(), output.toString(), conf);
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
-
- }
-
- public void updateKylinTableExd(String tableName, String outPath, Configuration config) throws IOException {
- List<String> columns = null;
- try {
- columns = readLines(new Path(outPath), config);
- } catch (Exception e) {
- logger.error("Failed to resolve cardinality for " + tableName + " from " + outPath, e);
- return;
- }
-
- StringBuffer cardi = new StringBuffer();
- Iterator<String> it = columns.iterator();
- while (it.hasNext()) {
- String string = it.next();
- String[] ss = StringUtils.split(string, "\t");
-
- if (ss.length != 2) {
- logger.info("The hadoop cardinality value is not valid " + string);
- continue;
- }
- cardi.append(ss[1]);
- cardi.append(",");
- }
- String scardi = cardi.toString();
- if (scardi.length() > 0) {
- scardi = scardi.substring(0, scardi.length() - 1);
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
- tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
- metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
- } else {
- throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
- }
- }
-
- private static List<String> readLines(Path location, Configuration conf) throws Exception {
- FileSystem fileSystem = FileSystem.get(location.toUri(), conf);
- CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- FileStatus[] items = fileSystem.listStatus(location);
- if (items == null)
- return new ArrayList<String>();
- List<String> results = new ArrayList<String>();
- for (FileStatus item : items) {
-
- // ignoring files like _SUCCESS
- if (item.getPath().getName().startsWith("_")) {
- continue;
- }
-
- CompressionCodec codec = factory.getCodec(item.getPath());
- InputStream stream = null;
-
- // check if we have a compression codec we need to use
- if (codec != null) {
- stream = codec.createInputStream(fileSystem.open(item.getPath()));
- } else {
- stream = fileSystem.open(item.getPath());
- }
-
- StringWriter writer = new StringWriter();
- IOUtils.copy(stream, writer, "UTF-8");
- String raw = writer.toString();
- for (String str : raw.split("\n")) {
- results.add(str);
- }
- }
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJob.java
deleted file mode 100644
index b600213..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJob.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * @author honma
- *
- */
-
-public class BaseCuboidJob extends CuboidJob {
- public BaseCuboidJob() {
- this.setMapperClass(BaseCuboidMapper.class);
- }
-
- public static void main(String[] args) throws Exception {
- CuboidJob job = new BaseCuboidJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapper.java
deleted file mode 100644
index d06963b..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapper.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesSplitter;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author George Song (ysong1)
- */
-public class BaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, Text> {
-
- private static final Logger logger = LoggerFactory.getLogger(BaseCuboidMapper.class);
-
- public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
- public static final byte[] ONE = Bytes.toBytes("1");
-
- private String cubeName;
- private String segmentName;
- private Cuboid baseCuboid;
- private CubeInstance cube;
- private CubeDesc cubeDesc;
- private CubeSegment cubeSegment;
- private List<byte[]> nullBytes;
-
- private CubeJoinedFlatTableDesc intermediateTableDesc;
- private String intermediateTableRowDelimiter;
- private byte byteRowDelimiter;
-
- private int counter;
- private int errorRecordCounter;
- private Text outputKey = new Text();
- private Text outputValue = new Text();
- protected MeasureIngester<?>[] aggrIngesters;
- protected Map<TblColRef, Dictionary<String>> dictionaryMap;
- private Object[] measures;
- private byte[][] keyBytesBuf;
- private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- private BytesSplitter bytesSplitter;
- private AbstractRowKeyEncoder rowKeyEncoder;
- private MeasureCodec measureCodec;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
- segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
- intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
- if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
- throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
- }
-
- byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
-
- cube = CubeManager.getInstance(config).getCube(cubeName);
- cubeDesc = cube.getDescriptor();
- cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
-
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
-
- bytesSplitter = new BytesSplitter(200, 16384);
- rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
-
- measureCodec = new MeasureCodec(cubeDesc.getMeasures());
- measures = new Object[cubeDesc.getMeasures().size()];
-
- int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
- keyBytesBuf = new byte[colCount][];
-
- aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
- dictionaryMap = cubeSegment.buildDictionaryMap();
-
- initNullBytes();
- }
-
- private void initNullBytes() {
- nullBytes = Lists.newArrayList();
- nullBytes.add(HIVE_NULL);
- String[] nullStrings = cubeDesc.getNullStrings();
- if (nullStrings != null) {
- for (String s : nullStrings) {
- nullBytes.add(Bytes.toBytes(s));
- }
- }
- }
-
- private boolean isNull(byte[] v) {
- for (byte[] nullByte : nullBytes) {
- if (Bytes.equals(v, nullByte))
- return true;
- }
- return false;
- }
-
- private byte[] buildKey(SplittedBytes[] splitBuffers) {
- int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
- for (int i = 0; i < baseCuboid.getColumns().size(); i++) {
- int index = rowKeyColumnIndexes[i];
- keyBytesBuf[i] = Arrays.copyOf(splitBuffers[index].value, splitBuffers[index].length);
- if (isNull(keyBytesBuf[i])) {
- keyBytesBuf[i] = null;
- }
- }
- return rowKeyEncoder.encode(keyBytesBuf);
- }
-
- private void buildValue(SplittedBytes[] splitBuffers) {
-
- for (int i = 0; i < measures.length; i++) {
- measures[i] = buildValueOf(i, splitBuffers);
- }
-
- valueBuf.clear();
- measureCodec.encode(measures, valueBuf);
- }
-
- private Object buildValueOf(int idxOfMeasure, SplittedBytes[] splitBuffers) {
- MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
- FunctionDesc function = measure.getFunction();
- int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
-
- int paramCount = function.getParameterCount();
- String[] inputToMeasure = new String[paramCount];
-
- // pick up parameter values
- ParameterDesc param = function.getParameter();
- int colParamIdx = 0; // index among parameters of column type
- for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
- String value;
- if (function.isCount()) {
- value = "1";
- } else if (param.isColumnType()) {
- value = getCell(colIdxOnFlatTable[colParamIdx++], splitBuffers);
- } else {
- value = param.getValue();
- }
- inputToMeasure[i] = value;
- }
-
- return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
- }
-
- private String getCell(int i, SplittedBytes[] splitBuffers) {
- byte[] bytes = Arrays.copyOf(splitBuffers[i].value, splitBuffers[i].length);
- if (isNull(bytes))
- return null;
- else
- return Bytes.toString(bytes);
- }
-
- @Override
- public void map(KEYIN key, Text value, Context context) throws IOException, InterruptedException {
- counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
- logger.info("Handled " + counter + " records!");
- }
-
- try {
- bytesSplitter.split(value.getBytes(), value.getLength(), byteRowDelimiter);
- intermediateTableDesc.sanityCheck(bytesSplitter);
-
- byte[] rowKey = buildKey(bytesSplitter.getSplitBuffers());
- outputKey.set(rowKey, 0, rowKey.length);
-
- buildValue(bytesSplitter.getSplitBuffers());
- outputValue.set(valueBuf.array(), 0, valueBuf.position());
-
- context.write(outputKey, outputValue);
- } catch (Exception ex) {
- handleErrorRecord(bytesSplitter, ex);
- }
- }
-
- private void handleErrorRecord(BytesSplitter bytesSplitter, Exception ex) throws IOException {
-
- ex.printStackTrace(System.err);
- System.err.println("Insane record: " + bytesSplitter);
-
- errorRecordCounter++;
- if (errorRecordCounter > BatchConstants.ERROR_RECORD_THRESHOLD) {
- if (ex instanceof IOException)
- throw (IOException) ex;
- else if (ex instanceof RuntimeException)
- throw (RuntimeException) ex;
- else
- throw new RuntimeException("", ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
deleted file mode 100644
index 3c1e4a5..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
-import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author George Song (ysong1)
- */
-public class CubeHFileJob extends AbstractHadoopJob {
-
- protected static final Logger logger = LoggerFactory.getLogger(CubeHFileJob.class);
-
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- parseOptions(options, args);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
-
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- CubeInstance cube = cubeMgr.getCube(cubeName);
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
-
- setJobClasspath(job);
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
- FileOutputFormat.setOutputPath(job, output);
-
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(CubeHFileMapper.class);
- job.setReducerClass(KeyValueSortReducer.class);
-
- // set job configuration
- job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- Configuration conf = HBaseConfiguration.create(getConf());
- // add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
- HTable htable = new HTable(conf, tableName);
-
- //Automatic config !
- HFileOutputFormat.configureIncrementalLoad(job, htable);
-
- // set block replication to 3 for hfiles
- conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- logger.error("error in CubeHFileJob", e);
- printUsage(options);
- throw e;
- } finally {
- if (job != null)
- cleanupTempConfFile(job.getConfiguration());
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new CubeHFileJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
deleted file mode 100644
index f12d229..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeHFileMapper extends KylinMapper<Text, Text, ImmutableBytesWritable, KeyValue> {
-
- ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
-
- String cubeName;
- CubeDesc cubeDesc;
-
- MeasureCodec inputCodec;
- Object[] inputMeasures;
- List<KeyValueCreator> keyValueCreators;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
-
- CubeManager cubeMgr = CubeManager.getInstance(config);
- cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();
-
- inputCodec = new MeasureCodec(cubeDesc.getMeasures());
- inputMeasures = new Object[cubeDesc.getMeasures().size()];
- keyValueCreators = Lists.newArrayList();
-
- for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
- for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
- keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
- }
- }
- }
-
- @Override
- public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- outputKey.set(key.getBytes(), 0, key.getLength());
- KeyValue outputValue;
-
- int n = keyValueCreators.size();
- if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for
- // simple full copy
-
- outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength());
- context.write(outputKey, outputValue);
-
- } else { // normal (complex) case that distributes measures to multiple
- // HBase columns
-
- inputCodec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), inputMeasures);
-
- for (int i = 0; i < n; i++) {
- outputValue = keyValueCreators.get(i).create(key, inputMeasures);
- context.write(outputKey, outputValue);
- }
- }
- }
-
- class KeyValueCreator {
- byte[] cfBytes;
- byte[] qBytes;
- long timestamp;
-
- int[] refIndex;
- MeasureDesc[] refMeasures;
-
- MeasureCodec codec;
- Object[] colValues;
- ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- boolean isFullCopy;
-
- public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {
-
- cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
- qBytes = Bytes.toBytes(colDesc.getQualifier());
- timestamp = 0; // use 0 for timestamp
-
- List<MeasureDesc> measures = cubeDesc.getMeasures();
- String[] measureNames = getMeasureNames(cubeDesc);
- String[] refs = colDesc.getMeasureRefs();
-
- refIndex = new int[refs.length];
- refMeasures = new MeasureDesc[refs.length];
- for (int i = 0; i < refs.length; i++) {
- refIndex[i] = indexOf(measureNames, refs[i]);
- refMeasures[i] = measures.get(refIndex[i]);
- }
-
- codec = new MeasureCodec(refMeasures);
- colValues = new Object[refs.length];
-
- isFullCopy = true;
- for (int i = 0; i < measures.size(); i++) {
- if (refIndex.length <= i || refIndex[i] != i)
- isFullCopy = false;
- }
- }
-
- public KeyValue create(Text key, Object[] measureValues) {
- for (int i = 0; i < colValues.length; i++) {
- colValues[i] = measureValues[refIndex[i]];
- }
-
- valueBuf.clear();
- codec.encode(colValues, valueBuf);
-
- return create(key, valueBuf.array(), 0, valueBuf.position());
- }
-
- public KeyValue create(Text key, byte[] value, int voffset, int vlen) {
- return new KeyValue(key.getBytes(), 0, key.getLength(), //
- cfBytes, 0, cfBytes.length, //
- qBytes, 0, qBytes.length, //
- timestamp, Type.Put, //
- value, voffset, vlen);
- }
-
- private int indexOf(String[] measureNames, String ref) {
- for (int i = 0; i < measureNames.length; i++)
- if (measureNames[i].equalsIgnoreCase(ref))
- return i;
-
- throw new IllegalArgumentException("Measure '" + ref + "' not found in " + Arrays.toString(measureNames));
- }
-
- private String[] getMeasureNames(CubeDesc cubeDesc) {
- List<MeasureDesc> measures = cubeDesc.getMeasures();
- String[] result = new String[measures.size()];
- for (int i = 0; i < measures.size(); i++)
- result[i] = measures.get(i).getName();
- return result;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
deleted file mode 100644
index e4875e9..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.CuboidCLI;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author ysong1
- */
-public class CuboidJob extends AbstractHadoopJob {
-
- protected static final Logger logger = LoggerFactory.getLogger(CuboidJob.class);
- private static final String MAPRED_REDUCE_TASKS = "mapred.reduce.tasks";
-
- @SuppressWarnings("rawtypes")
- private Class<? extends Mapper> mapperClass;
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_SEGMENT_NAME);
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_NCUBOID_LEVEL);
- options.addOption(OPTION_INPUT_FORMAT);
- parseOptions(options, args);
-
- Path input = new Path(getOptionValue(OPTION_INPUT_PATH));
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
- int nCuboidLevel = Integer.parseInt(getOptionValue(OPTION_NCUBOID_LEVEL));
- String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeMgr = CubeManager.getInstance(config);
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- logger.info("Starting: " + job.getJobName());
- FileInputFormat.setInputPaths(job, input);
-
- setJobClasspath(job);
-
- // Mapper
- if (this.mapperClass == null) {
- throw new Exception("Mapper class is not set!");
- }
-
- boolean isInputTextFormat = false;
- if (hasOption(OPTION_INPUT_FORMAT) && ("textinputformat".equalsIgnoreCase(getOptionValue(OPTION_INPUT_FORMAT)))) {
- isInputTextFormat = true;
- }
-
- if (isInputTextFormat) {
- job.setInputFormatClass(TextInputFormat.class);
-
- } else {
- job.setInputFormatClass(SequenceFileInputFormat.class);
- }
- job.setMapperClass(this.mapperClass);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Text.class);
- job.setCombinerClass(CuboidReducer.class); // for base cuboid shuffle skew, some rowkey aggregates far more records than others
-
- // Reducer
- job.setReducerClass(CuboidReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
-
- FileOutputFormat.setOutputPath(job, output);
-
- // set job configuration
- job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- job.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, nCuboidLevel);
-
- // add metadata to distributed cache
- attachKylinPropsAndMetadata(cube, job.getConfiguration());
-
- setReduceTaskNum(job, config, cubeName, nCuboidLevel);
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- logger.error("error in CuboidJob", e);
- printUsage(options);
- throw e;
- } finally {
- if (job != null)
- cleanupTempConfFile(job.getConfiguration());
- }
- }
-
- protected void setReduceTaskNum(Job job, KylinConfig config, String cubeName, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
- Configuration jobConf = job.getConfiguration();
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
-
- CubeDesc cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
-
- double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
- double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
-
- // total map input MB
- double totalMapInputMB = this.getTotalMapInputMB();
-
- // output / input ratio
- int preLevelCuboids, thisLevelCuboids;
- if (level == 0) { // base cuboid
- preLevelCuboids = thisLevelCuboids = 1;
- } else { // n-cuboid
- int[] allLevelCount = CuboidCLI.calculateAllLevelCount(cubeDesc);
- preLevelCuboids = allLevelCount[level - 1];
- thisLevelCuboids = allLevelCount[level];
- }
-
- // total reduce input MB
- double totalReduceInputMB = totalMapInputMB * thisLevelCuboids / preLevelCuboids;
-
- // number of reduce tasks
- int numReduceTasks = (int) Math.round(totalReduceInputMB / perReduceInputMB * reduceCountRatio);
-
- // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
- if (cubeDesc.hasMemoryHungryMeasures()) {
- numReduceTasks = numReduceTasks * 4;
- }
-
- // at least 1 reducer
- numReduceTasks = Math.max(1, numReduceTasks);
- // no more than 5000 reducer by default
- numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
-
- jobConf.setInt(MAPRED_REDUCE_TASKS, numReduceTasks);
-
- logger.info("Having total map input MB " + Math.round(totalMapInputMB));
- logger.info("Having level " + level + ", pre-level cuboids " + preLevelCuboids + ", this level cuboids " + thisLevelCuboids);
- logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio);
- logger.info("Setting " + MAPRED_REDUCE_TASKS + "=" + numReduceTasks);
- }
-
- /**
- * @param mapperClass
- * the mapperClass to set
- */
- @SuppressWarnings("rawtypes")
- public void setMapperClass(Class<? extends Mapper> mapperClass) {
- this.mapperClass = mapperClass;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidReducer.java
deleted file mode 100644
index 3859d0e..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidReducer.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.measure.MeasureAggregators;
-import org.apache.kylin.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
-
- private static final Logger logger = LoggerFactory.getLogger(CuboidReducer.class);
-
- private String cubeName;
- private CubeDesc cubeDesc;
- private List<MeasureDesc> measuresDescs;
-
- private MeasureCodec codec;
- private MeasureAggregators aggs;
-
- private int counter;
- private int cuboidLevel;
- private boolean[] needAggr;
- private Object[] input;
- private Object[] result;
-
- private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private Text outputValue = new Text();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
- // only used in Build job, not in Merge job
- cuboidLevel = context.getConfiguration().getInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 0);
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
-
- cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
- measuresDescs = cubeDesc.getMeasures();
-
- codec = new MeasureCodec(measuresDescs);
- aggs = new MeasureAggregators(measuresDescs);
-
- input = new Object[measuresDescs.size()];
- result = new Object[measuresDescs.size()];
- needAggr = new boolean[measuresDescs.size()];
-
- if (cuboidLevel > 0) {
- for (int i = 0; i < measuresDescs.size(); i++) {
- needAggr[i] = !measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
- }
- }
- }
-
- @Override
- public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
-
- aggs.reset();
-
- for (Text value : values) {
- codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input);
- if (cuboidLevel > 0) {
- aggs.aggregate(input, needAggr);
- } else {
- aggs.aggregate(input);
- }
- }
- aggs.collectStates(result);
-
- valueBuf.clear();
- codec.encode(result, valueBuf);
-
- outputValue.set(valueBuf.array(), 0, valueBuf.position());
- context.write(key, outputValue);
-
- counter++;
- if (counter % BatchConstants.COUNTER_MAX == 0) {
- logger.info("Handled " + counter + " records!");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
deleted file mode 100644
index 9792463..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.util.HashSet;
-
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * @author yangli9
- */
-public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable, Text, ShortWritable, Text> {
-
- private Text outputValue = new Text();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- }
-
- @Override
- public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
-
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- }
-
- for (ByteArray value : set) {
- outputValue.set(value.data);
- context.write(key, outputValue);
- }
- }
-
-}
[32/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
deleted file mode 100644
index 0af7dfc..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.File;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.LookupBytesTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testBasic() throws Exception {
- TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES");
- TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings");
- LookupBytesTable lookup;
-
- System.out.println("============================================================================");
-
- File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv");
- lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://" + f.getAbsolutePath(), 10));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv");
- lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new FileTable("file://" + f.getAbsolutePath(), 36));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- ByteArray k1 = new ByteArray(Bytes.toBytes("533"));
- ByteArray k2 = new ByteArray(Bytes.toBytes("0"));
- Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 });
- System.out.println(lookup.getRow(key));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
deleted file mode 100644
index 4647ef8..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.math.BigDecimal;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- *
- */
-public class NumberDictionaryTest {
-
- NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec();
- Random rand = new Random();
-
- @Test
- public void testNumberEncode() {
- checkCodec("12345", "00000000000012345");
- checkCodec("12345.123", "00000000000012345.123");
- checkCodec("-12345", "-9999999999987654;");
- checkCodec("-12345.123", "-9999999999987654.876;");
- checkCodec("0", "00000000000000000");
- checkCodec("0.0", "00000000000000000.0");
- }
-
- private void checkCodec(String number, String code) {
- assertEquals(code, encodeNumber(number));
- assertEquals(number, decodeNumber(code));
- }
-
- private String decodeNumber(String code) {
- byte[] buf = Bytes.toBytes(code);
- System.arraycopy(buf, 0, codec.buf, 0, buf.length);
- codec.bufOffset = 0;
- codec.bufLen = buf.length;
- int len = codec.decodeNumber(buf, 0);
- return Bytes.toString(buf, 0, len);
- }
-
- private String encodeNumber(String number) {
- byte[] num1 = Bytes.toBytes(number);
- codec.encodeNumber(num1, 0, num1.length);
- return Bytes.toString(codec.buf, codec.bufOffset, codec.bufLen);
- }
-
- @Test
- public void testDictionary() {
- int n = 100;
-
- Set<BigDecimal> set = Sets.newHashSet();
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
- for (int i = 0; i < n; i++) {
- String num = randNumber();
- if (set.add(new BigDecimal(num))) {
- builder.addValue(num);
- }
- }
-
- List<BigDecimal> sorted = Lists.newArrayList();
- sorted.addAll(set);
- Collections.sort(sorted);
-
- // test exact match
- NumberDictionary<String> dict = builder.build(0);
- for (int i = 0; i < sorted.size(); i++) {
- String dictNum = dict.getValueFromId(i);
- System.out.println(sorted.get(i) + "\t" + dictNum);
- }
-
- for (int i = 0; i < sorted.size(); i++) {
- String dictNum = dict.getValueFromId(i);
- assertEquals(sorted.get(i), new BigDecimal(dictNum));
- }
-
- // test rounding
- for (int i = 0; i < n; i++) {
- String randStr = randNumber();
- BigDecimal rand = new BigDecimal(randStr);
- int binarySearch = Collections.binarySearch(sorted, rand);
- if (binarySearch >= 0)
- continue;
- int insertion = -(binarySearch + 1);
- int expectedLowerId = insertion - 1;
- int expectedHigherId = insertion;
- // System.out.println("-- " + randStr + ", " + expectedLowerId +
- // ", " + expectedHigherId);
-
- if (expectedLowerId < 0) {
- try {
- dict.getIdFromValue(randStr, -1);
- fail();
- } catch (IllegalArgumentException ex) {
- // expect
- }
- } else {
- assertEquals(expectedLowerId, dict.getIdFromValue(randStr, -1));
- }
-
- if (expectedHigherId >= sorted.size()) {
- try {
- dict.getIdFromValue(randStr, 1);
- fail();
- } catch (IllegalArgumentException ex) {
- // expect
- }
- } else {
- assertEquals(expectedHigherId, dict.getIdFromValue(randStr, 1));
- }
- }
- }
-
- private String randNumber() {
- int digits1 = rand.nextInt(10);
- int digits2 = rand.nextInt(3);
- int sign = rand.nextInt(2);
- if (digits1 == 0 && digits2 == 0) {
- return randNumber();
- }
- StringBuilder buf = new StringBuilder();
- if (sign == 1)
- buf.append("-");
- for (int i = 0; i < digits1; i++)
- buf.append("" + rand.nextInt(10));
- if (digits2 > 0) {
- buf.append(".");
- for (int i = 0; i < digits2; i++)
- buf.append("" + rand.nextInt(9) + 1); // BigDecimal thinks 4.5
- // != 4.50, my god!
- }
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index c8dddde..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
- SnapshotManager snapshotMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- snapshotMgr = SnapshotManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() throws Exception {
- String tableName = "EDW.TEST_SITES";
- HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
- snapshotMgr.wipeoutCache();
-
- SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
- // compare hive & snapshot
- TableReader hiveReader = hiveTable.getReader();
- TableReader snapshotReader = snapshot.getReader();
-
- while (true) {
- boolean hiveNext = hiveReader.next();
- boolean snapshotNext = snapshotReader.next();
- assertEquals(hiveNext, snapshotNext);
-
- if (hiveNext == false)
- break;
-
- String[] hiveRow = hiveReader.getRow();
- String[] snapshotRow = snapshotReader.getRow();
- assertArrayEquals(hiveRow, snapshotRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
deleted file mode 100644
index 0d1999a..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.FileTableReader;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class TableReaderTest {
-
- @Test
- public void testBasicReader() throws IOException {
- File f = new File("src/test/resources/dict/DW_SITES");
- FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), FileTable.DELIM_AUTO, 10);
- while (reader.next()) {
- assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow()));
- break;
- }
- reader.close();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
deleted file mode 100644
index cb15731..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.TreeSet;
-
-import org.junit.Test;
-
-public class TrieDictionaryTest {
-
- public static void main(String[] args) throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- // InputStream is =
- // Util.getPackageResourceAsStream(TrieDictionaryTest.class,
- // "eng_com.dic");
- ArrayList<String> str = loadStrings(is);
- benchmarkStringDictionary(str);
- }
-
- @Test
- public void partOverflowTest() {
- ArrayList<String> str = new ArrayList<String>();
- // str.add("");
- str.add("part");
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
- System.out.println("The length of the long string is " + longStr.length());
- str.add(longStr);
-
- str.add("zzzzzz" + longStr);// another long string
-
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- TrieDictionary<String> dict = b.build(0);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // test serialize
- dict = testSerialize(dict);
-
- // test basic id<==>value
- Iterator<String> it = set.iterator();
- int id = 0;
- int previousId = -1;
- for (; it.hasNext(); id++) {
- String value = it.next();
-
- // in case of overflow parts, there exist interpolation nodes
- // they exist to make sure that any node's part is shorter than 255
- int actualId = dict.getIdFromValue(value);
- assertTrue(actualId >= id);
- assertTrue(actualId > previousId);
- previousId = actualId;
-
- assertEquals(value, dict.getValueFromId(actualId));
- }
- }
-
- @Test
- public void emptyValueTest() {
- ArrayList<String> str = new ArrayList<String>();
- str.add("");
- str.add("part");
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
- testStringDictionary(str, null);
- }
-
- @Test
- public void simpleTrieTest() {
- ArrayList<String> str = new ArrayList<String>();
- str.add("part");
- str.add("part"); // meant to be dup
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
-
- ArrayList<String> notFound = new ArrayList<String>();
- notFound.add("");
- notFound.add("p");
- notFound.add("pa");
- notFound.add("pb");
- notFound.add("parti");
- notFound.add("partz");
- notFound.add("partyz");
-
- testStringDictionary(str, notFound);
- }
-
- @Test
- public void englishWordsTest() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic");
- ArrayList<String> str = loadStrings(is);
- testStringDictionary(str, null);
- }
-
- @Test
- public void categoryNamesTest() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- ArrayList<String> str = loadStrings(is);
- testStringDictionary(str, null);
- }
-
- private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException {
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- b.stats().print();
- TrieDictionary<String> dict = b.build(0);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // prepare id==>value array and value==>id map
- HashMap<String, Integer> map = new HashMap<String, Integer>();
- String[] strArray = new String[set.size()];
- byte[][] array = new byte[set.size()][];
- Iterator<String> it = set.iterator();
- for (int id = 0; it.hasNext(); id++) {
- String value = it.next();
- map.put(value, id);
- strArray[id] = value;
- array[id] = value.getBytes("UTF-8");
- }
-
- // System.out.println("Dict size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(dict));
- // System.out.println("Map size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(map));
- // System.out.println("Array size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(strArray));
-
- // warm-up, said that code only got JIT after run 1k-10k times,
- // following jvm options may help
- // -XX:CompileThreshold=1500
- // -XX:+PrintCompilation
- benchmark("Warm up", dict, set, map, strArray, array);
- benchmark("Benchmark", dict, set, map, strArray, array);
- }
-
- private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
- int n = set.size();
- int times = 10 * 1000 * 1000 / n; // run 10 million lookups
- int keep = 0; // make sure JIT don't OPT OUT function calls under test
- byte[] valueBytes = new byte[dict.getSizeOfValue()];
- long start;
-
- // benchmark value==>id, via HashMap
- System.out.println(msg + " HashMap lookup value==>id");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= map.get(strArray[j]);
- }
- }
- long timeValueToIdByMap = System.currentTimeMillis() - start;
- System.out.println(timeValueToIdByMap);
-
- // benchmark value==>id, via Dict
- System.out.println(msg + " Dictionary lookup value==>id");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
- }
- }
- long timeValueToIdByDict = System.currentTimeMillis() - start;
- System.out.println(timeValueToIdByDict);
-
- // benchmark id==>value, via Array
- System.out.println(msg + " Array lookup id==>value");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= strArray[j].length();
- }
- }
- long timeIdToValueByArray = System.currentTimeMillis() - start;
- System.out.println(timeIdToValueByArray);
-
- // benchmark id==>value, via Dict
- System.out.println(msg + " Dictionary lookup id==>value");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
- }
- }
- long timeIdToValueByDict = System.currentTimeMillis() - start;
- System.out.println(timeIdToValueByDict);
-
- return keep;
- }
-
- private static void testStringDictionary(ArrayList<String> str, ArrayList<String> notFound) {
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- int baseId = new Random().nextInt(100);
- TrieDictionary<String> dict = b.build(baseId);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // test serialize
- dict = testSerialize(dict);
-
- // test basic id<==>value
- Iterator<String> it = set.iterator();
- int id = baseId;
- for (; it.hasNext(); id++) {
- String value = it.next();
- // System.out.println("checking " + id + " <==> " + value);
-
- assertEquals(id, dict.getIdFromValue(value));
- assertEquals(value, dict.getValueFromId(id));
- }
- if (notFound != null) {
- for (String s : notFound) {
- try {
- dict.getIdFromValue(s);
- fail("For not found value '" + s + "', IllegalArgumentException is expected");
- } catch (IllegalArgumentException e) {
- // good
- }
- }
- }
-
- // test null value
- int nullId = dict.getIdFromValue(null);
- assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
- }
-
- private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream dataout = new DataOutputStream(bout);
- dict.write(dataout);
- dataout.close();
- ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
- DataInputStream datain = new DataInputStream(bin);
- TrieDictionary<String> r = new TrieDictionary<String>();
- r.readFields(datain);
- datain.close();
- return r;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) {
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- for (String s : str)
- b.addValue(s);
- return b;
- }
-
- private static ArrayList<String> loadStrings(InputStream is) throws Exception {
- ArrayList<String> r = new ArrayList<String>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
- try {
- String word;
- while ((word = reader.readLine()) != null) {
- word = word.trim();
- if (word.isEmpty() == false)
- r.add(word);
- }
- } finally {
- reader.close();
- is.close();
- }
- return r;
- }
-
- @Test
- public void testSuperLongStringValue() {
- String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
-
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
- String v2 = longPrefix + "xyz";
-
- b.addValue(v1);
- b.addValue(v2);
- TrieDictionary<String> dict = b.build(0);
- dict.dump(System.out);
- }
-
- @Test
- public void testRounding() {
- // see NumberDictionaryTest.testRounding();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
deleted file mode 100644
index 3d7267d..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 2/3/15.
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
- private KylinConfig config = null;
-
- private LookupTable lookupTable;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- config = KylinConfig.getInstanceFromEnv();
- lookupTable = initLookupTable();
- }
-
- @After
- public void tearDown() {
- cleanupTestMetadata();
- }
-
- @Test
- public void testScan() throws Exception {
- List<String> values = new ArrayList<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- List<String> results = lookupTable.scan("CAL_DT", values, "YEAR_BEG_DT");
-
- Assert.assertTrue(results.size() > 0);
- for (String i : results) {
- System.out.println(i);
-
- Assert.assertEquals("2012-01-01", i);
- }
- }
-
- @Test
- public void testMapRange() throws Exception {
- List<String> values = new ArrayList<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- Pair<String, String> results = lookupTable.mapRange("CAL_DT", "2012-01-24", "2012-12-30", "QTR_BEG_DT");
-
- Assert.assertTrue(results != null);
- System.out.println("The first qtr_beg_dt is " + results.getFirst());
- System.out.println("The last qtr_beg_dt is " + results.getSecond());
-
- Assert.assertEquals("2012-01-01", results.getFirst());
- Assert.assertEquals("2012-10-01", results.getSecond());
- }
-
- @Test
- public void testMapValues() throws Exception {
- Set<String> values = new HashSet<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- Set<String> results = lookupTable.mapValues("CAL_DT", values, "YEAR_BEG_DT");
-
- Assert.assertTrue(results.size() == 1);
- for (String i : results) {
- System.out.println(i);
-
- Assert.assertEquals("2012-01-01", i);
- }
- }
-
- public LookupTable initLookupTable() throws Exception {
-
- MetadataManager metaMgr = MetadataManager.getInstance(config);
-
- String tableName = "EDW.TEST_CAL_DT";
- String[] pkCols = new String[] { "CAL_DT" };
- String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot";
- SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = metaMgr.getTableDesc(tableName);
- LookupTable lt = new LookupStringTable(tableDesc, pkCols, snapshot);
-
- System.out.println(lt);
-
- return lt;
- }
-
- private SnapshotManager getSnapshotManager() {
- return SnapshotManager.getInstance(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/DW_SITES
----------------------------------------------------------------------
diff --git a/dictionary/src/test/resources/dict/DW_SITES b/dictionary/src/test/resources/dict/DW_SITES
deleted file mode 100644
index 6f6af46..0000000
Binary files a/dictionary/src/test/resources/dict/DW_SITES and /dev/null differ
[38/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
deleted file mode 100644
index ca69ada..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ /dev/null
@@ -1,830 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.net.util.Base64;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.CaseInsensitiveStringMap;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeDesc extends RootPersistentEntity {
-
- public static enum DeriveType {
- LOOKUP, PK_FK
- }
-
- public static class DeriveInfo {
- public DeriveType type;
- public DimensionDesc dimension;
- public TblColRef[] columns;
- public boolean isOneToOne; // only used when ref from derived to host
-
- DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
- this.type = type;
- this.dimension = dimension;
- this.columns = columns;
- this.isOneToOne = isOneToOne;
- }
-
- @Override
- public String toString() {
- return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
- }
-
- }
-
- private KylinConfig config;
- private DataModelDesc model;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("model_name")
- private String modelName;
- @JsonProperty("description")
- private String description;
- @JsonProperty("null_string")
- private String[] nullStrings;
- @JsonProperty("dimensions")
- private List<DimensionDesc> dimensions;
- @JsonProperty("measures")
- private List<MeasureDesc> measures;
- @JsonProperty("rowkey")
- private RowKeyDesc rowkey;
- @JsonProperty("hbase_mapping")
- private HBaseMappingDesc hbaseMapping;
- @JsonProperty("signature")
- private String signature;
- @JsonProperty("notify_list")
- private List<String> notifyList;
- @JsonProperty("auto_merge_time_ranges")
- private long[] autoMergeTimeRanges;
- @JsonProperty("retention_range")
- private long retentionRange = 0;
-
- private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
- private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
- private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
-
- /**
- * Error messages during resolving json metadata
- */
- private List<String> errors = new ArrayList<String>();
-
- /**
- * @return all columns this cube can support, including derived
- */
- public Set<TblColRef> listAllColumns() {
- return allColumns;
- }
-
- /**
- * @return dimension columns including derived, BUT NOT measures
- */
- public Set<TblColRef> listDimensionColumnsIncludingDerived() {
- return dimensionColumns;
- }
-
- /**
- * @return dimension columns excluding derived and measures
- */
- public List<TblColRef> listDimensionColumnsExcludingDerived() {
- List<TblColRef> result = new ArrayList<TblColRef>();
- for (TblColRef col : dimensionColumns) {
- if (isDerived(col) == false)
- result.add(col);
- }
- return result;
- }
-
- /**
- * Find FunctionDesc by Full Expression.
- *
- * @return
- */
- public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
- for (MeasureDesc m : measures) {
- if (m.getFunction().equals(manualFunc))
- return m.getFunction();
- }
- return null;
- }
-
- public TblColRef findColumnRef(String table, String column) {
- Map<String, TblColRef> cols = columnMap.get(table);
- if (cols == null)
- return null;
- else
- return cols.get(column);
- }
-
- public DimensionDesc findDimensionByColumn(TblColRef col) {
- for (DimensionDesc dim : dimensions) {
- if (ArrayUtils.contains(dim.getColumnRefs(), col))
- return dim;
- }
- return null;
- }
-
- public DimensionDesc findDimensionByTable(String lookupTableName) {
- lookupTableName = lookupTableName.toUpperCase();
- for (DimensionDesc dim : dimensions)
- if (dim.getTable() != null && dim.getTable().equals(lookupTableName))
- return dim;
- return null;
- }
-
- public DimensionDesc findDimensionByName(String dimName) {
- dimName = dimName.toUpperCase();
- for (DimensionDesc dim : dimensions) {
- if (dimName.equals(dim.getName()))
- return dim;
- }
- return null;
- }
-
- /**
- * Get all functions from each measure.
- *
- * @return
- */
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measures) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
- /**
- * @deprecated use getModel().getAllTables() instead
- * @return
- */
- public List<TableDesc> listTables() {
- MetadataManager metaMgr = MetadataManager.getInstance(config);
- HashSet<String> tableNames = new HashSet<String>();
- List<TableDesc> result = new ArrayList<TableDesc>();
-
- tableNames.add(this.getFactTable().toUpperCase());
- for (DimensionDesc dim : dimensions) {
- String table = dim.getTable();
- if (table != null)
- tableNames.add(table.toUpperCase());
- }
-
- for (String tableName : tableNames) {
- result.add(metaMgr.getTableDesc(tableName));
- }
-
- return result;
- }
-
- public boolean isDerived(TblColRef col) {
- return derivedToHostMap.containsKey(col);
- }
-
- public DeriveInfo getHostInfo(TblColRef derived) {
- return derivedToHostMap.get(derived);
- }
-
- public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
- Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
- for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
- Array<TblColRef> hostCols = entry.getKey();
- boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
- if (!hostOnRow)
- continue;
-
- List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
- for (DeriveInfo info : entry.getValue()) {
- if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
- // any
- // wanted
- // columns?
- wantedInfo.add(info);
- }
-
- if (wantedInfo.size() > 0)
- result.put(hostCols, wantedInfo);
- }
- return result;
- }
-
- public String getResourcePath() {
- return getCubeDescResourcePath(name);
- }
-
- public static String getCubeDescResourcePath(String descName) {
- return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
- }
-
- // ============================================================================
-
- public HBaseMappingDesc getHBaseMapping() {
- return hbaseMapping;
- }
-
- public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getModelName() {
- return modelName;
- }
-
- public void setModelName(String modelName) {
- this.modelName = modelName;
- }
-
- public DataModelDesc getModel() {
- return model;
- }
-
- public void setModel(DataModelDesc model) {
- this.model = model;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public String getFactTable() {
- return model.getFactTable().toUpperCase();
- }
-
- public String[] getNullStrings() {
- return nullStrings;
- }
-
- public List<DimensionDesc> getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(List<DimensionDesc> dimensions) {
- this.dimensions = dimensions;
- }
-
- public List<MeasureDesc> getMeasures() {
- return measures;
- }
-
- public void setMeasures(List<MeasureDesc> measures) {
- this.measures = measures;
- }
-
- public RowKeyDesc getRowkey() {
- return rowkey;
- }
-
- public void setRowkey(RowKeyDesc rowkey) {
- this.rowkey = rowkey;
- }
-
- public String getSignature() {
- return signature;
- }
-
- public void setSignature(String signature) {
- this.signature = signature;
- }
-
- public List<String> getNotifyList() {
- return notifyList;
- }
-
- public void setNotifyList(List<String> notifyList) {
- this.notifyList = notifyList;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- CubeDesc cubeDesc = (CubeDesc) o;
-
- if (!name.equals(cubeDesc.name))
- return false;
- if (!getFactTable().equals(cubeDesc.getFactTable()))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = 0;
- result = 31 * result + name.hashCode();
- result = 31 * result + getFactTable().hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "CubeDesc [name=" + name + "]";
- }
-
- public String calculateSignature() {
- MessageDigest md = null;
- try {
- md = MessageDigest.getInstance("MD5");
- StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
-
- byte[] signature = md.digest(sigString.toString().getBytes());
- return new String(Base64.encodeBase64(signature));
- } catch (NoSuchAlgorithmException | JsonProcessingException e) {
- throw new RuntimeException("Failed to calculate signature");
- }
- }
-
- public Map<String, TblColRef> buildColumnNameAbbreviation() {
- Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>();
- for (TblColRef col : listDimensionColumnsExcludingDerived()) {
- r.put(col.getName(), col);
- }
- return r;
- }
-
- public void init(KylinConfig config, Map<String, TableDesc> tables) {
- this.errors.clear();
- this.config = config;
-
- if (this.modelName == null || this.modelName.length() == 0) {
- this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
- }
-
- this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
-
- if (this.model == null) {
- this.addError("No data model found with name '" + modelName + "'.");
- }
-
- for (DimensionDesc dim : dimensions) {
- dim.init(this, tables);
- }
-
- sortDimAndMeasure();
- initDimensionColumns(tables);
- initMeasureColumns(tables);
-
- rowkey.init(this);
- if (hbaseMapping != null) {
- hbaseMapping.init(this);
- }
-
- initMeasureReferenceToColumnFamily();
-
- // check all dimension columns are presented on rowkey
- List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
- if (rowkey.getRowKeyColumns().length != dimCols.size()) {
- addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "); rowkeys are " + Arrays.toString(rowkey.getRowKeyColumns()) + ", dimension columns are " + dimCols.toString());
- }
- }
-
- private void initDimensionColumns(Map<String, TableDesc> tables) {
- for (DimensionDesc dim : dimensions) {
- JoinDesc join = dim.getJoin();
-
- // init dimension columns
- ArrayList<TblColRef> dimCols = Lists.newArrayList();
- String[] colStrs = dim.getColumn();
-
- // when column is omitted, special case
- if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
- for (TblColRef col : join.getForeignKeyColumns()) {
- dimCols.add(initDimensionColRef(col));
- }
- }
- // normal case
- else {
- if (colStrs == null || colStrs.length == 0)
- throw new IllegalStateException("Dimension column must not be blank " + dim);
-
- for (String colStr : colStrs) {
- dimCols.add(initDimensionColRef(dim, colStr));
- }
-
- // fill back column ref in hierarchy
- if (dim.isHierarchy()) {
- for (int i = 0; i < dimCols.size(); i++)
- dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
- }
- }
-
- TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
- dim.setColumnRefs(dimColArray);
-
- // init derived columns
- TblColRef[] hostCols = dimColArray;
- if (dim.isDerived()) {
- String[] derived = dim.getDerived();
- String[][] split = splitDerivedColumnAndExtra(derived);
- String[] derivedNames = split[0];
- String[] derivedExtra = split[1];
- TblColRef[] derivedCols = new TblColRef[derivedNames.length];
- for (int i = 0; i < derivedNames.length; i++) {
- derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
- }
- initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
- }
-
- // PK-FK derive the other side
- if (join != null) {
- TblColRef[] fk = join.getForeignKeyColumns();
- TblColRef[] pk = join.getPrimaryKeyColumns();
-
- allColumns.addAll(Arrays.asList(fk));
- allColumns.addAll(Arrays.asList(pk));
- for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, fk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
-
- /** disable this code as we don't need fk be derived from pk
- for (int i = 0; i < pk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, pk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(fk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
- */
- }
- }
- }
-
- private String[][] splitDerivedColumnAndExtra(String[] derived) {
- String[] cols = new String[derived.length];
- String[] extra = new String[derived.length];
- for (int i = 0; i < derived.length; i++) {
- String str = derived[i];
- int cut = str.indexOf(":");
- if (cut >= 0) {
- cols[i] = str.substring(0, cut);
- extra[i] = str.substring(cut + 1).trim();
- } else {
- cols[i] = str;
- extra[i] = "";
- }
- }
- return new String[][] { cols, extra };
- }
-
- private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) {
- initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null);
- }
-
- private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
- if (hostCols.length == 0 || derivedCols.length == 0)
- throw new IllegalStateException("host/derived columns must not be empty");
-
- // Although FK derives PK automatically, user unaware of this can declare PK as derived dimension explicitly.
- // In that case, derivedCols[] will contain a FK which is transformed from the PK by initDimensionColRef().
- // Must drop FK from derivedCols[] before continue.
- for (int i = 0; i < derivedCols.length; i++) {
- if (ArrayUtils.contains(hostCols, derivedCols[i])) {
- derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
- extra = (String[]) ArrayUtils.remove(extra, i);
- i--;
- }
- }
-
- Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
- List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
- if (infoList == null) {
- hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
- }
- infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
-
- for (int i = 0; i < derivedCols.length; i++) {
- TblColRef derivedCol = derivedCols[i];
- boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
- }
- }
-
- private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
- TableDesc table = dim.getTableDesc();
- ColumnDesc col = table.findColumnByName(colName);
- if (col == null)
- throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
-
- TblColRef ref = new TblColRef(col);
-
- // always use FK instead PK, FK could be shared by more than one lookup tables
- JoinDesc join = dim.getJoin();
- if (join != null) {
- int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), ref);
- if (idx >= 0) {
- ref = join.getForeignKeyColumns()[idx];
- }
- }
- return initDimensionColRef(ref);
- }
-
- private TblColRef initDimensionColRef(TblColRef ref) {
- TblColRef existing = findColumnRef(ref.getTable(), ref.getName());
- if (existing != null) {
- return existing;
- }
-
- allColumns.add(ref);
- dimensionColumns.add(ref);
-
- Map<String, TblColRef> cols = columnMap.get(ref.getTable());
- if (cols == null) {
- columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>());
- }
- cols.put(ref.getName(), ref);
- return ref;
- }
-
- private void initMeasureColumns(Map<String, TableDesc> tables) {
- if (measures == null || measures.isEmpty()) {
- return;
- }
-
- TableDesc factTable = tables.get(getFactTable());
- for (MeasureDesc m : measures) {
- m.setName(m.getName().toUpperCase());
-
- if (m.getDependentMeasureRef() != null) {
- m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
- }
-
- FunctionDesc f = m.getFunction();
- f.setExpression(f.getExpression().toUpperCase());
- f.initReturnDataType();
-
- for (ParameterDesc p = f.getParameter(); p != null; p = p.getNextParameter()) {
- p.setValue(p.getValue().toUpperCase());
- }
-
- ArrayList<TblColRef> colRefs = Lists.newArrayList();
- for (ParameterDesc p = f.getParameter(); p != null; p = p.getNextParameter()) {
- if (p.isColumnType()) {
- ColumnDesc sourceColumn = factTable.findColumnByName(p.getValue());
- TblColRef colRef = new TblColRef(sourceColumn);
- colRefs.add(colRef);
- allColumns.add(colRef);
- }
- }
-
- f.getParameter().setColRefs(colRefs);
-
- }
- }
-
- private void initMeasureReferenceToColumnFamily() {
- if (measures == null || measures.size() == 0)
- return;
-
- Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
- for (MeasureDesc m : measures)
- measureCache.put(m.getName(), m);
-
- for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
- for (HBaseColumnDesc c : cf.getColumns()) {
- MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
- for (int i = 0; i < c.getMeasureRefs().length; i++) {
- measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
- }
- c.setMeasures(measureDescs);
- c.setColumnFamilyName(cf.getName());
- }
- }
- }
-
- private void sortDimAndMeasure() {
- sortDimensionsByID();
- sortMeasuresByID();
- for (DimensionDesc dim : dimensions) {
- sortHierarchiesByLevel(dim.getHierarchy());
- }
- }
-
- private void sortDimensionsByID() {
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc d1, DimensionDesc d2) {
- Integer id1 = d1.getId();
- Integer id2 = d2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortMeasuresByID() {
- if (measures == null) {
- measures = Lists.newArrayList();
- }
-
- Collections.sort(measures, new Comparator<MeasureDesc>() {
- @Override
- public int compare(MeasureDesc m1, MeasureDesc m2) {
- Integer id1 = m1.getId();
- Integer id2 = m2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
- if (hierarchies != null) {
- Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
- @Override
- public int compare(HierarchyDesc h1, HierarchyDesc h2) {
- Integer level1 = Integer.parseInt(h1.getLevel());
- Integer level2 = Integer.parseInt(h2.getLevel());
- return level1.compareTo(level2);
- }
- });
- }
- }
-
- /**
- * Add error info and thrown exception out
- *
- * @param message
- */
- public void addError(String message) {
- addError(message, false);
- }
-
- /**
- * @param message
- * error message
- * @param silent
- * if throw exception
- */
- public void addError(String message, boolean silent) {
- if (!silent) {
- throw new IllegalStateException(message);
- } else {
- this.errors.add(message);
- }
- }
-
- public List<String> getError() {
- return this.errors;
- }
-
- public HBaseMappingDesc getHbaseMapping() {
- return hbaseMapping;
- }
-
- public void setHbaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
- public void setNullStrings(String[] nullStrings) {
- this.nullStrings = nullStrings;
- }
-
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
-
- public long[] getAutoMergeTimeRanges() {
- return autoMergeTimeRanges;
- }
-
- public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
- this.autoMergeTimeRanges = autoMergeTimeRanges;
- }
-
- public List<TblColRef> getAllColumnsNeedDictionary() {
- List<TblColRef> result = Lists.newArrayList();
-
- for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) {
- TblColRef colRef = rowKeyColDesc.getColRef();
- if (rowkey.isUseDictionary(colRef)) {
- result.add(colRef);
- }
- }
-
- for (MeasureDesc measure : measures) {
- MeasureType<?> aggrType = measure.getFunction().getMeasureType();
- result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
- }
- return result;
- }
-
- public boolean hasMemoryHungryMeasures() {
- for (MeasureDesc measure : measures) {
- if (measure.getFunction().getMeasureType().isMemoryHungry()) {
- return true;
- }
- }
- return false;
- }
-
- public static CubeDesc getCopyOf(CubeDesc cubeDesc) {
- CubeDesc newCubeDesc = new CubeDesc();
- newCubeDesc.setName(cubeDesc.getName());
- newCubeDesc.setModelName(cubeDesc.getModelName());
- newCubeDesc.setDescription(cubeDesc.getDescription());
- newCubeDesc.setNullStrings(cubeDesc.getNullStrings());
- newCubeDesc.setDimensions(cubeDesc.getDimensions());
- newCubeDesc.setMeasures(cubeDesc.getMeasures());
- newCubeDesc.setRowkey(cubeDesc.getRowkey());
- newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping());
- newCubeDesc.setSignature(cubeDesc.getSignature());
- newCubeDesc.setNotifyList(cubeDesc.getNotifyList());
- newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges());
- newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange());
- newCubeDesc.setConfig(cubeDesc.getConfig());
- newCubeDesc.updateRandomUuid();
- return newCubeDesc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
deleted file mode 100644
index 9df6669..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc {
-
- @JsonProperty("id")
- private int id;
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("hierarchy")
- private boolean isHierarchy;
- @JsonProperty("table")
- private String table;
- @JsonProperty("column")
- private String[] column;
- @JsonProperty("derived")
- private String[] derived;
-
- private TableDesc tableDesc;
- private JoinDesc join;
- private HierarchyDesc[] hierarchy;
-
- // computed
- private TblColRef[] columnRefs;
- private TblColRef[] derivedColRefs;
-
- public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
- if (name != null)
- name = name.toUpperCase();
-
- if (table != null)
- table = table.toUpperCase();
-
- tableDesc = tables.get(this.getTable());
- if (tableDesc == null)
- throw new IllegalStateException("Can't find table " + table + " for dimension " + name);
-
- join = null;
- for (LookupDesc lookup : cubeDesc.getModel().getLookups()) {
- if (lookup.getTable().equalsIgnoreCase(this.getTable())) {
- join = lookup.getJoin();
- break;
- }
- }
-
- if (isHierarchy && this.column.length > 0) {
- List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
- for (int i = 0, n = this.column.length; i < n; i++) {
- String aColumn = this.column[i];
- HierarchyDesc aHierarchy = new HierarchyDesc();
- aHierarchy.setLevel(String.valueOf(i + 1));
- aHierarchy.setColumn(aColumn);
- hierarchyList.add(aHierarchy);
- }
-
- this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
- }
-
- if (hierarchy != null && hierarchy.length == 0)
- hierarchy = null;
- if (derived != null && derived.length == 0)
- derived = null;
-
- if (hierarchy != null) {
- for (HierarchyDesc h : hierarchy)
- h.setColumn(h.getColumn().toUpperCase());
- }
-
- if (derived != null) {
- StringUtil.toUpperCaseArray(derived, derived);
- }
-
- if (derived != null && join == null) {
- throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
- }
- }
-
- public boolean isHierarchyColumn(TblColRef col) {
- if (hierarchy == null)
- return false;
-
- for (HierarchyDesc hier : hierarchy) {
- if (hier.getColumnRef().equals(col))
- return true;
- }
- return false;
- }
-
- public boolean isDerived() {
- return derived != null;
- }
-
- public boolean isHierarchy() {
- return isHierarchy;
- }
-
- public void setHierarchy(boolean isHierarchy) {
- this.isHierarchy = isHierarchy;
- }
-
- public String getTable() {
- return table;
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
- public JoinDesc getJoin() {
- return join;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public TblColRef[] getColumnRefs() {
- return this.columnRefs;
- }
-
- public void setColumnRefs(TblColRef[] colRefs) {
- this.columnRefs = colRefs;
- }
-
- public String[] getColumn() {
- return this.column;
- }
-
- public void setColumn(String[] column) {
- this.column = column;
- }
-
- public HierarchyDesc[] getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(HierarchyDesc[] hierarchy) {
- this.hierarchy = hierarchy;
- }
-
- public String[] getDerived() {
- return derived;
- }
-
- public void setDerived(String[] derived) {
- this.derived = derived;
- }
-
- public TblColRef[] getDerivedColRefs() {
- return derivedColRefs;
- }
-
- public void setDerivedColRefs(TblColRef[] derivedColRefs) {
- this.derivedColRefs = derivedColRefs;
- }
-
- public TableDesc getTableDesc() {
- return this.tableDesc;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- DimensionDesc that = (DimensionDesc) o;
-
- if (id != that.id)
- return false;
- if (!name.equals(that.name))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = id;
- result = 31 * result + name.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
deleted file mode 100644
index 1387fa5..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/30/13 Time: 10:57 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnDesc {
-
- @JsonProperty("qualifier")
- private String qualifier;
- @JsonProperty("measure_refs")
- private String[] measureRefs;
-
- // these two will be assemble in runtime.
- private MeasureDesc[] measures;
- private String columnFamilyName;
-
- public String getQualifier() {
- return qualifier;
- }
-
- public void setQualifier(String qualifier) {
- this.qualifier = qualifier;
- }
-
- public String[] getMeasureRefs() {
- return measureRefs;
- }
-
- public void setMeasureRefs(String[] measureRefs) {
- this.measureRefs = measureRefs;
- }
-
- public MeasureDesc[] getMeasures() {
- return measures;
- }
-
- public int findMeasureIndex(FunctionDesc function) {
- for (int i = 0; i < measures.length; i++) {
- if (measures[i].getFunction().equals(function)) {
- return i;
- }
- }
- return -1;
- }
-
- public void setMeasures(MeasureDesc[] measures) {
- this.measures = measures;
- }
-
- public String getColumnFamilyName() {
- return columnFamilyName;
- }
-
- public void setColumnFamilyName(String columnFamilyName) {
- this.columnFamilyName = columnFamilyName;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((columnFamilyName == null) ? 0 : columnFamilyName.hashCode());
- result = prime * result + ((qualifier == null) ? 0 : qualifier.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HBaseColumnDesc other = (HBaseColumnDesc) obj;
- if (columnFamilyName == null) {
- if (other.columnFamilyName != null)
- return false;
- } else if (!columnFamilyName.equals(other.columnFamilyName))
- return false;
- if (qualifier == null) {
- if (other.qualifier != null)
- return false;
- } else if (!qualifier.equals(other.qualifier))
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "HBaseColumnDesc [qualifier=" + qualifier + ", measureRefs=" + Arrays.toString(measureRefs) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
deleted file mode 100644
index c1d552a..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.util.Arrays;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/30/13 Time: 10:41 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnFamilyDesc {
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("columns")
- private HBaseColumnDesc[] columns;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public HBaseColumnDesc[] getColumns() {
- return columns;
- }
-
- public void setColumns(HBaseColumnDesc[] columns) {
- this.columns = columns;
- }
-
- @Override
- public String toString() {
- return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
deleted file mode 100644
index 2e73723..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:44 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseMappingDesc {
-
- @JsonProperty("column_family")
- private HBaseColumnFamilyDesc[] columnFamily;
-
- // point to the cube instance which contain this HBaseMappingDesc instance.
- private CubeDesc cubeRef;
-
- public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
- Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
- HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
- if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
- return result;
- }
- for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) {
- for (HBaseColumnDesc c : cf.getColumns()) {
- for (MeasureDesc m : c.getMeasures()) {
- if (m.getFunction().equals(function)) {
- result.add(c);
- }
- }
- }
- }
- return result;
- }
-
- public CubeDesc getCubeRef() {
- return cubeRef;
- }
-
- public void setCubeRef(CubeDesc cubeRef) {
- this.cubeRef = cubeRef;
- }
-
- public HBaseColumnFamilyDesc[] getColumnFamily() {
- return columnFamily;
- }
-
- public void setColumnFamily(HBaseColumnFamilyDesc[] columnFamily) {
- this.columnFamily = columnFamily;
- }
-
- public void init(CubeDesc cubeDesc) {
- cubeRef = cubeDesc;
-
- for (HBaseColumnFamilyDesc cf : columnFamily) {
- cf.setName(cf.getName().toUpperCase());
-
- for (HBaseColumnDesc c : cf.getColumns()) {
- c.setQualifier(c.getQualifier().toUpperCase());
- StringUtil.toUpperCaseArray(c.getMeasureRefs(), c.getMeasureRefs());
- }
- }
- }
-
- @Override
- public String toString() {
- return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
deleted file mode 100644
index a834810..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:46 AM To
- * change this template use File | Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HierarchyDesc {
-
- @JsonProperty("level")
- private String level;
- @JsonProperty("column")
- private String column;
-
- private TblColRef columnRef;
-
- public String getLevel() {
- return level;
- }
-
- public void setLevel(String level) {
- this.level = level;
- }
-
- public TblColRef getColumnRef() {
- return columnRef;
- }
-
- public void setColumnRef(TblColRef column) {
- this.columnRef = column;
- }
-
- public String getColumn() {
- return column;
- }
-
- public void setColumn(String columnName) {
- this.column = columnName;
- }
-
- @Override
- public String toString() {
- return "HierarchyDesc [level=" + level + ", column=" + column + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
deleted file mode 100644
index 11a0bc2..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- *
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyColDesc {
-
- @JsonProperty("column")
- private String column;
- @JsonProperty("length")
- private int length;
- @JsonProperty("dictionary")
- private String dictionary;
- @JsonProperty("mandatory")
- private boolean mandatory = false;
-
- // computed
- private int bitIndex;
- private TblColRef colRef;
-
- public String getDictionary() {
- return dictionary;
- }
-
- public String getColumn() {
- return column;
- }
-
- public void setColumn(String column) {
- this.column = column;
- }
-
- public int getLength() {
- return length;
- }
-
- public boolean isMandatory() {
- return mandatory;
- }
-
- public int getBitIndex() {
- return bitIndex;
- }
-
- void setBitIndex(int index) {
- this.bitIndex = index;
- }
-
- public TblColRef getColRef() {
- return colRef;
- }
-
- void setColRef(TblColRef colRef) {
- this.colRef = colRef;
- }
-
- public void setDictionary(String dictionary) {
- this.dictionary = dictionary;
- }
-
- @Override
- public String toString() {
- return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
deleted file mode 100644
index 1ce68f5..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created by lukhan on 1/2/14.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyDesc {
-
- public static class HierarchyMask {
- public long fullMask;
- public long[] allMasks;
- }
-
- public static class AggrGroupMask {
- public AggrGroupMask(int size) {
- groupOneBitMasks = new long[size];
- }
-
- public long groupMask;
- public long groupOneBitMasks[];
- public long uniqueMask;
- public long leftoverMask;
- }
-
- @JsonProperty("rowkey_columns")
- private RowKeyColDesc[] rowkeyColumns;
- @JsonProperty("aggregation_groups")
- private String[][] aggregationGroups;
-
- // computed content
- private CubeDesc cubeDesc;
- private Map<TblColRef, RowKeyColDesc> columnMap;
-
- private long fullMask;
- private long mandatoryColumnMask;
- private AggrGroupMask[] aggrGroupMasks;
- private long aggrGroupFullMask;
- private long tailMask;
-
- private List<HierarchyMask> hierarchyMasks;
-
- public RowKeyColDesc[] getRowKeyColumns() {
- return rowkeyColumns;
- }
-
- // search a specific row key col
- public int getRowKeyIndexByColumnName(String columnName) {
- if (this.rowkeyColumns == null)
- return -1;
-
- for (int i = 0; i < this.rowkeyColumns.length; ++i) {
- RowKeyColDesc desc = this.rowkeyColumns[i];
- if (desc.getColumn().equalsIgnoreCase(columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- public int getNCuboidBuildLevels() {
- // N aggregation columns requires N levels of cuboid build
- // - N columns requires N-1 levels build
- // - zero tail cuboid needs one more additional level
- Set<String> aggDims = new HashSet<String>();
- for (String[] aggrGroup : aggregationGroups) {
- for (String dim : aggrGroup) {
- aggDims.add(dim);
- }
- }
- return aggDims.size();
- }
-
- public String[][] getAggregationGroups() {
- return aggregationGroups;
- }
-
- public CubeDesc getCubeRef() {
- return cubeDesc;
- }
-
- public void setCubeRef(CubeDesc cubeRef) {
- this.cubeDesc = cubeRef;
- }
-
- public long getFullMask() {
- return fullMask;
- }
-
- public long getMandatoryColumnMask() {
- return mandatoryColumnMask;
- }
-
- public long getAggrGroupFullMask() {
- return aggrGroupFullMask;
- }
-
- public AggrGroupMask[] getAggrGroupMasks() {
- return aggrGroupMasks;
- }
-
- public List<HierarchyMask> getHierarchyMasks() {
- return hierarchyMasks;
- }
-
- public long getTailMask() {
- return tailMask;
- }
-
- public int getColumnBitIndex(TblColRef col) {
- return getColDesc(col).getBitIndex();
- }
-
- public int getColumnLength(TblColRef col) {
- return getColDesc(col).getLength();
- }
-
- public String getDictionary(TblColRef col) {
- return getColDesc(col).getDictionary();
- }
-
- private RowKeyColDesc getColDesc(TblColRef col) {
- RowKeyColDesc desc = columnMap.get(col);
-
- if (desc == null)
- throw new NullPointerException("Column " + col + " does not exist in row key desc");
- return desc;
- }
-
- public boolean isUseDictionary(TblColRef col) {
- String useDictionary = getDictionary(col);
- return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary);
- }
-
- public void init(CubeDesc cube) {
- setCubeRef(cube);
- Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation();
-
- buildRowKey(colNameAbbr);
- buildAggregationGroups(colNameAbbr);
- buildHierarchyMasks();
- }
-
- @Override
- public String toString() {
- return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]";
- }
-
- private void buildRowKey(Map<String, TblColRef> colNameAbbr) {
- columnMap = new HashMap<TblColRef, RowKeyColDesc>();
- mandatoryColumnMask = 0;
-
- for (int i = 0; i < rowkeyColumns.length; i++) {
- RowKeyColDesc rowKeyColDesc = rowkeyColumns[i];
- String column = rowKeyColDesc.getColumn();
- rowKeyColDesc.setColumn(column.toUpperCase());
- rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1);
- rowKeyColDesc.setColRef(colNameAbbr.get(column));
- if (rowKeyColDesc.getColRef() == null) {
- throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc);
- }
-
- columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc);
-
- if (rowKeyColDesc.isMandatory()) {
- mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex();
- }
- }
- }
-
- private void buildAggregationGroups(Map<String, TblColRef> colNameAbbr) {
- if (aggregationGroups == null) {
- aggregationGroups = new String[0][];
- }
-
- for (int i = 0; i < aggregationGroups.length; i++) {
- StringUtil.toUpperCaseArray(aggregationGroups[i], this.aggregationGroups[i]);
- }
-
- for (int i = 0; i < this.rowkeyColumns.length; i++) {
- int index = rowkeyColumns[i].getBitIndex();
- this.fullMask |= 1L << index;
- }
-
- this.aggrGroupMasks = new AggrGroupMask[aggregationGroups.length];
- for (int i = 0; i < this.aggregationGroups.length; i++) {
- String[] aggGrp = this.aggregationGroups[i];
- AggrGroupMask mask = new AggrGroupMask(aggGrp.length);
-
- for (int j = 0; j < aggGrp.length; j++) {
- TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase());
- if (aggCol == null) {
- throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName());
- }
- Integer index = getColumnBitIndex(aggCol);
- mask.groupMask |= 1L << index;
- mask.groupOneBitMasks[j] = 1L << index;
- this.aggrGroupFullMask |= 1L << index;
- }
- this.aggrGroupMasks[i] = mask;
- }
-
- this.tailMask = fullMask ^ mandatoryColumnMask ^ aggrGroupFullMask;
-
- // unique mask = (bits in this group) - (bits in following groups)
- // leftover mask = (tail bits) + (bits in following groups) - (bits in
- // this group)
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- AggrGroupMask mask = aggrGroupMasks[i];
-
- mask.uniqueMask = mask.groupMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.uniqueMask &= ~aggrGroupMasks[j].groupMask;
- }
-
- mask.leftoverMask = tailMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.leftoverMask |= aggrGroupMasks[j].groupMask;
- }
- mask.leftoverMask &= ~mask.groupMask;
- }
- }
-
- private void buildHierarchyMasks() {
- this.hierarchyMasks = new ArrayList<HierarchyMask>();
-
- for (DimensionDesc dimension : this.cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- if (hierarchies == null || hierarchies.length == 0)
- continue;
-
- HierarchyMask mask = new HierarchyMask();
- ArrayList<Long> allMaskList = new ArrayList<Long>();
- for (int i = 0; i < hierarchies.length; i++) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((tailMask & bit) > 0)
- continue; // ignore levels in tail, they don't participate
- // aggregation group combination anyway
-
- mask.fullMask |= bit;
- allMaskList.add(mask.fullMask);
- }
-
- mask.allMasks = new long[allMaskList.size()];
- for (int i = 0; i < allMaskList.size(); i++)
- mask.allMasks[i] = allMaskList.get(i);
-
- this.hierarchyMasks.add(mask);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeBuildTypeEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeBuildTypeEnum.java b/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeBuildTypeEnum.java
deleted file mode 100644
index bbd9a33..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/model/v1/CubeBuildTypeEnum.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v1;
-
-/**
- * @author xduo
- *
- */
-public enum CubeBuildTypeEnum {
- /**
- * rebuild a segment or incremental build
- */
- BUILD,
- /**
- * merge segments
- */
- MERGE;
-}
[44/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Bytes.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Bytes.java b/common/src/main/java/org/apache/kylin/common/util/Bytes.java
deleted file mode 100644
index 0bb0af2..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Bytes.java
+++ /dev/null
@@ -1,2203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkPositionIndex;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.charset.Charset;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.security.SecureRandom;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-
-import sun.misc.Unsafe;
-
-/**
- * Utility class that handles byte arrays, conversions to/from other types,
- * comparisons, hash code generation, manufacturing keys for HashMaps or
- * HashSets, etc.
- *
- * Copied from org.apache.hadoop.hbase.util.Bytes
- */
-public class Bytes {
- //HConstants.UTF8_ENCODING should be updated if this changed
- /**
- * When we encode strings, we always specify UTF8 encoding
- */
- private static final String UTF8_ENCODING = "UTF-8";
-
- //HConstants.UTF8_CHARSET should be updated if this changed
- /**
- * When we encode strings, we always specify UTF8 encoding
- */
- private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
-
- //HConstants.EMPTY_BYTE_ARRAY should be updated if this changed
- private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
-
- private static final Log LOG = LogFactory.getLog(Bytes.class);
-
- /**
- * Size of boolean in bytes
- */
- public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
-
- /**
- * Size of byte in bytes
- */
- public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
-
- /**
- * Size of char in bytes
- */
- public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
-
- /**
- * Size of double in bytes
- */
- public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
-
- /**
- * Size of float in bytes
- */
- public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
-
- /**
- * Size of int in bytes
- */
- public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
-
- /**
- * Size of long in bytes
- */
- public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
-
- /**
- * Size of short in bytes
- */
- public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
-
- /**
- * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
- * Estimate based on study of jhat and jprofiler numbers.
- */
- // JHat says BU is 56 bytes.
- // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
- public static final int ESTIMATED_HEAP_TAX = 16;
-
- /**
- * Returns length of the byte array, returning 0 if the array is null.
- * Useful for calculating sizes.
- *
- * @param b byte array, which can be null
- * @return 0 if b is null, otherwise returns length
- */
- final public static int len(byte[] b) {
- return b == null ? 0 : b.length;
- }
-
- /**
- * Byte array comparator class.
- */
- public static class ByteArrayComparator implements RawComparator<byte[]> {
- /**
- * Constructor
- */
- public ByteArrayComparator() {
- super();
- }
-
- @Override
- public int compare(byte[] left, byte[] right) {
- return compareTo(left, right);
- }
-
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return LexicographicalComparerHolder.BEST_COMPARER.compareTo(b1, s1, l1, b2, s2, l2);
- }
- }
-
- /**
- * A {@link ByteArrayComparator} that treats the empty array as the largest value.
- * This is useful for comparing row end keys for regions.
- */
- // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region
- // boundaries. Thus semantically, we should treat empty byte array as the smallest value
- // while comparing row keys, start keys etc; but as the largest value for comparing
- // region boundaries for endKeys.
- public static class RowEndKeyComparator extends ByteArrayComparator {
- @Override
- public int compare(byte[] left, byte[] right) {
- return compare(left, 0, left.length, right, 0, right.length);
- }
-
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- if (b1 == b2 && s1 == s2 && l1 == l2) {
- return 0;
- }
- if (l1 == 0) {
- return l2; //0 or positive
- }
- if (l2 == 0) {
- return -1;
- }
- return super.compare(b1, s1, l1, b2, s2, l2);
- }
- }
-
- /**
- * Pass this to TreeMaps where byte [] are keys.
- */
- public final static Comparator<byte[]> BYTES_COMPARATOR = new ByteArrayComparator();
-
- /**
- * Use comparing byte arrays, byte-by-byte
- */
- public final static RawComparator<byte[]> BYTES_RAWCOMPARATOR = new ByteArrayComparator();
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix.
- *
- * @param in Input to read from.
- * @return byte array read off <code>in</code>
- * @throws java.io.IOException e
- */
- public static byte[] readByteArray(final DataInput in) throws IOException {
- int len = WritableUtils.readVInt(in);
- if (len < 0) {
- throw new NegativeArraySizeException(Integer.toString(len));
- }
- byte[] result = new byte[len];
- in.readFully(result, 0, len);
- return result;
- }
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix.
- * IOException is converted to a RuntimeException.
- *
- * @param in Input to read from.
- * @return byte array read off <code>in</code>
- */
- public static byte[] readByteArrayThrowsRuntime(final DataInput in) {
- try {
- return readByteArray(in);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Write byte-array with a WritableableUtils.vint prefix.
- *
- * @param out output stream to be written to
- * @param b array to write
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b) throws IOException {
- if (b == null) {
- WritableUtils.writeVInt(out, 0);
- } else {
- writeByteArray(out, b, 0, b.length);
- }
- }
-
- /**
- * Write byte-array to out with a vint length prefix.
- *
- * @param out output stream
- * @param b array
- * @param offset offset into array
- * @param length length past offset
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b, final int offset, final int length) throws IOException {
- WritableUtils.writeVInt(out, length);
- out.write(b, offset, length);
- }
-
- /**
- * Write byte-array from src to tgt with a vint length prefix.
- *
- * @param tgt target array
- * @param tgtOffset offset into target array
- * @param src source array
- * @param srcOffset source offset
- * @param srcLength source length
- * @return New offset in src array.
- */
- public static int writeByteArray(final byte[] tgt, final int tgtOffset, final byte[] src, final int srcOffset, final int srcLength) {
- byte[] vint = vintToBytes(srcLength);
- System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
- int offset = tgtOffset + vint.length;
- System.arraycopy(src, srcOffset, tgt, offset, srcLength);
- return offset + srcLength;
- }
-
- /**
- * Put bytes at the specified byte array position.
- *
- * @param tgtBytes the byte array
- * @param tgtOffset position in the array
- * @param srcBytes array to write out
- * @param srcOffset source offset
- * @param srcLength source length
- * @return incremented offset
- */
- public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset, int srcLength) {
- System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
- return tgtOffset + srcLength;
- }
-
- /**
- * Write a single byte out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param b byte to write out
- * @return incremented offset
- */
- public static int putByte(byte[] bytes, int offset, byte b) {
- bytes[offset] = b;
- return offset + 1;
- }
-
- /**
- * Add the whole content of the ByteBuffer to the bytes arrays. The ByteBuffer is modified.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param buf ByteBuffer to write out
- * @return incremented offset
- */
- public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
- int len = buf.remaining();
- buf.get(bytes, offset, len);
- return offset + len;
- }
-
- /**
- * Returns a new byte array, copied from the given {@code buf},
- * from the index 0 (inclusive) to the limit (exclusive),
- * regardless of the current position.
- * The position and the other index parameters are not changed.
- *
- * @param buf a byte buffer
- * @return the byte array
- * @see #getBytes(ByteBuffer)
- */
- public static byte[] toBytes(ByteBuffer buf) {
- ByteBuffer dup = buf.duplicate();
- dup.position(0);
- return readBytes(dup);
- }
-
- private static byte[] readBytes(ByteBuffer buf) {
- byte[] result = new byte[buf.remaining()];
- buf.get(result);
- return result;
- }
-
- /**
- * @param b Presumed UTF-8 encoded byte array.
- * @return String made from <code>b</code>
- */
- public static String toString(final byte[] b) {
- if (b == null) {
- return null;
- }
- return toString(b, 0, b.length);
- }
-
- /**
- * Joins two byte arrays together using a separator.
- *
- * @param b1 The first byte array.
- * @param sep The separator to use.
- * @param b2 The second byte array.
- */
- public static String toString(final byte[] b1, String sep, final byte[] b2) {
- return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
- }
-
- /**
- * This method will convert utf8 encoded bytes into a string. If
- * the given byte array is null, this method will return null.
- *
- * @param b Presumed UTF-8 encoded byte array.
- * @param off offset into array
- * @param len length of utf-8 sequence
- * @return String made from <code>b</code> or null
- */
- public static String toString(final byte[] b, int off, int len) {
- if (b == null) {
- return null;
- }
- if (len == 0) {
- return "";
- }
- return new String(b, off, len, UTF8_CHARSET);
- }
-
- /**
- * Write a printable representation of a byte array.
- *
- * @param b byte array
- * @return string
- * @see #toStringBinary(byte[], int, int)
- */
- public static String toStringBinary(final byte[] b) {
- if (b == null)
- return "null";
- return toStringBinary(b, 0, b.length);
- }
-
- /**
- * Converts the given byte buffer to a printable representation,
- * from the index 0 (inclusive) to the limit (exclusive),
- * regardless of the current position.
- * The position and the other index parameters are not changed.
- *
- * @param buf a byte buffer
- * @return a string representation of the buffer's binary contents
- * @see #toBytes(ByteBuffer)
- * @see #getBytes(ByteBuffer)
- */
- public static String toStringBinary(ByteBuffer buf) {
- if (buf == null)
- return "null";
- if (buf.hasArray()) {
- return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
- }
- return toStringBinary(toBytes(buf));
- }
-
- /**
- * Write a printable representation of a byte array. Non-printable
- * characters are hex escaped in the format \\x%02X, eg:
- * \x00 \x05 etc
- *
- * @param b array to write out
- * @param off offset to start at
- * @param len length to write
- * @return string output
- */
- public static String toStringBinary(final byte[] b, int off, int len) {
- StringBuilder result = new StringBuilder();
- // Just in case we are passed a 'len' that is > buffer length...
- if (off >= b.length)
- return result.toString();
- if (off + len > b.length)
- len = b.length - off;
- for (int i = off; i < off + len; ++i) {
- int ch = b[i] & 0xFF;
- if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
- result.append((char) ch);
- } else {
- result.append(String.format("\\x%02X", ch));
- }
- }
- return result.toString();
- }
-
- private static boolean isHexDigit(char c) {
- return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
- }
-
- /**
- * Takes a ASCII digit in the range A-F0-9 and returns
- * the corresponding integer/ordinal value.
- *
- * @param ch The hex digit.
- * @return The converted hex value as a byte.
- */
- public static byte toBinaryFromHex(byte ch) {
- if (ch >= 'A' && ch <= 'F')
- return (byte) ((byte) 10 + (byte) (ch - 'A'));
- // else
- return (byte) (ch - '0');
- }
-
- public static byte[] toBytesBinary(String in) {
- // this may be bigger than we need, but let's be safe.
- byte[] b = new byte[in.length()];
- int size = 0;
- for (int i = 0; i < in.length(); ++i) {
- char ch = in.charAt(i);
- if (ch == '\\' && in.length() > i + 1 && in.charAt(i + 1) == 'x') {
- // ok, take next 2 hex digits.
- char hd1 = in.charAt(i + 2);
- char hd2 = in.charAt(i + 3);
-
- // they need to be A-F0-9:
- if (!isHexDigit(hd1) || !isHexDigit(hd2)) {
- // bogus escape code, ignore:
- continue;
- }
- // turn hex ASCII digit -> number
- byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
-
- b[size++] = d;
- i += 3; // skip 3
- } else {
- b[size++] = (byte) ch;
- }
- }
- // resize:
- byte[] b2 = new byte[size];
- System.arraycopy(b, 0, b2, 0, size);
- return b2;
- }
-
- /**
- * Converts a string to a UTF-8 byte array.
- *
- * @param s string
- * @return the byte array
- */
- public static byte[] toBytes(String s) {
- return s.getBytes(UTF8_CHARSET);
- }
-
- /**
- * Convert a boolean to a byte array. True becomes -1
- * and false becomes 0.
- *
- * @param b value
- * @return <code>b</code> encoded in a byte array.
- */
- public static byte[] toBytes(final boolean b) {
- return new byte[] { b ? (byte) -1 : (byte) 0 };
- }
-
- /**
- * Reverses {@link #toBytes(boolean)}
- *
- * @param b array
- * @return True or false.
- */
- public static boolean toBoolean(final byte[] b) {
- if (b.length != 1) {
- throw new IllegalArgumentException("Array has wrong size: " + b.length);
- }
- return b[0] != (byte) 0;
- }
-
- /**
- * Convert a long value to a byte array using big-endian.
- *
- * @param val value to convert
- * @return the byte array
- */
- public static byte[] toBytes(long val) {
- byte[] b = new byte[8];
- for (int i = 7; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a long value. Reverses
- * {@link #toBytes(long)}
- *
- * @param bytes array
- * @return the long value
- */
- public static long toLong(byte[] bytes) {
- return toLong(bytes, 0, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value. Assumes there will be
- * {@link #SIZEOF_LONG} bytes available.
- *
- * @param bytes bytes
- * @param offset offset
- * @return the long value
- */
- public static long toLong(byte[] bytes, int offset) {
- return toLong(bytes, offset, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value.
- *
- * @param bytes array of bytes
- * @param offset offset into array
- * @param length length of data (must be {@link #SIZEOF_LONG})
- * @return the long value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static long toLong(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_LONG || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return toLongUnsafe(bytes, offset);
- } else {
- long l = 0;
- for (int i = offset; i < offset + length; i++) {
- l <<= 8;
- l ^= bytes[i] & 0xFF;
- }
- return l;
- }
- }
-
- private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes, final int offset, final int length, final int expectedLength) {
- String reason;
- if (length != expectedLength) {
- reason = "Wrong length: " + length + ", expected " + expectedLength;
- } else {
- reason = "offset (" + offset + ") + length (" + length + ") exceed the" + " capacity of the array: " + bytes.length;
- }
- return new IllegalArgumentException(reason);
- }
-
- /**
- * Put a long value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val long to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putLong(byte[] bytes, int offset, long val) {
- if (bytes.length - offset < SIZEOF_LONG) {
- throw new IllegalArgumentException("Not enough room to put a long at" + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return putLongUnsafe(bytes, offset, val);
- } else {
- for (int i = offset + 7; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_LONG;
- }
- }
-
- /**
- * Put a long value out to the specified byte array position (Unsafe).
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val long to write out
- * @return incremented offset
- */
- public static int putLongUnsafe(byte[] bytes, int offset, long val) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- val = Long.reverseBytes(val);
- }
- org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putLong(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
- return offset + SIZEOF_LONG;
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes byte array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes) {
- return toFloat(bytes, 0);
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes array to convert
- * @param offset offset into array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param f float value
- * @return New offset in <code>bytes</code>
- */
- public static int putFloat(byte[] bytes, int offset, float f) {
- return putInt(bytes, offset, Float.floatToRawIntBits(f));
- }
-
- /**
- * @param f float value
- * @return the float represented as byte []
- */
- public static byte[] toBytes(final float f) {
- // Encode it as int
- return Bytes.toBytes(Float.floatToRawIntBits(f));
- }
-
- /**
- * @param bytes byte array
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes) {
- return toDouble(bytes, 0);
- }
-
- /**
- * @param bytes byte array
- * @param offset offset where double is
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param d value
- * @return New offset into array <code>bytes</code>
- */
- public static int putDouble(byte[] bytes, int offset, double d) {
- return putLong(bytes, offset, Double.doubleToLongBits(d));
- }
-
- /**
- * Serialize a double as the IEEE 754 double format output. The resultant
- * array will be 8 bytes long.
- *
- * @param d value
- * @return the double represented as byte []
- */
- public static byte[] toBytes(final double d) {
- // Encode it as a long
- return Bytes.toBytes(Double.doubleToRawLongBits(d));
- }
-
- /**
- * Convert an int value to a byte array. Big-endian. Same as what DataOutputStream.writeInt
- * does.
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(int val) {
- byte[] b = new byte[4];
- for (int i = 3; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @return the int value
- */
- public static int toInt(byte[] bytes) {
- return toInt(bytes, 0, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the int value
- */
- public static int toInt(byte[] bytes, int offset) {
- return toInt(bytes, offset, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length of int (has to be {@link #SIZEOF_INT})
- * @return the int value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static int toInt(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_INT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return toIntUnsafe(bytes, offset);
- } else {
- int n = 0;
- for (int i = offset; i < (offset + length); i++) {
- n <<= 8;
- n ^= bytes[i] & 0xFF;
- }
- return n;
- }
- }
-
- /**
- * Converts a byte array to an int value (Unsafe version)
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the int value
- */
- public static int toIntUnsafe(byte[] bytes, int offset) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- return Integer.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getInt(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
- } else {
- return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getInt(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- /**
- * Converts a byte array to an short value (Unsafe version)
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the short value
- */
- public static short toShortUnsafe(byte[] bytes, int offset) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- return Short.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getShort(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
- } else {
- return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getShort(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- /**
- * Converts a byte array to an long value (Unsafe version)
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the long value
- */
- public static long toLongUnsafe(byte[] bytes, int offset) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- return Long.reverseBytes(org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getLong(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET));
- } else {
- return org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.getLong(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length how many bytes should be considered for creating int
- * @return the int value
- * @throws IllegalArgumentException if there's not enough room in the array at the offset
- * indicated.
- */
- public static int readAsInt(byte[] bytes, int offset, final int length) {
- if (offset + length > bytes.length) {
- throw new IllegalArgumentException("offset (" + offset + ") + length (" + length + ") exceed the" + " capacity of the array: " + bytes.length);
- }
- int n = 0;
- for (int i = offset; i < (offset + length); i++) {
- n <<= 8;
- n ^= bytes[i] & 0xFF;
- }
- return n;
- }
-
- /**
- * Put an int value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val int to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putInt(byte[] bytes, int offset, int val) {
- if (bytes.length - offset < SIZEOF_INT) {
- throw new IllegalArgumentException("Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return putIntUnsafe(bytes, offset, val);
- } else {
- for (int i = offset + 3; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_INT;
- }
- }
-
- /**
- * Put an int value out to the specified byte array position (Unsafe).
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val int to write out
- * @return incremented offset
- */
- public static int putIntUnsafe(byte[] bytes, int offset, int val) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- val = Integer.reverseBytes(val);
- }
- org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putInt(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
- return offset + SIZEOF_INT;
- }
-
- /**
- * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(short val) {
- byte[] b = new byte[SIZEOF_SHORT];
- b[1] = (byte) val;
- val >>= 8;
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @return the short value
- */
- public static short toShort(byte[] bytes) {
- return toShort(bytes, 0, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the short value
- */
- public static short toShort(byte[] bytes, int offset) {
- return toShort(bytes, offset, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length, has to be {@link #SIZEOF_SHORT}
- * @return the short value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT}
- * or if there's not enough room in the array at the offset indicated.
- */
- public static short toShort(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_SHORT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return toShortUnsafe(bytes, offset);
- } else {
- short n = 0;
- n ^= bytes[offset] & 0xFF;
- n <<= 8;
- n ^= bytes[offset + 1] & 0xFF;
- return n;
- }
- }
-
- /**
- * Returns a new byte array, copied from the given {@code buf},
- * from the position (inclusive) to the limit (exclusive).
- * The position and the other index parameters are not changed.
- *
- * @param buf a byte buffer
- * @return the byte array
- * @see #toBytes(ByteBuffer)
- */
- public static byte[] getBytes(ByteBuffer buf) {
- return readBytes(buf.duplicate());
- }
-
- /**
- * Put a short value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val short to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putShort(byte[] bytes, int offset, short val) {
- if (bytes.length - offset < SIZEOF_SHORT) {
- throw new IllegalArgumentException("Not enough room to put a short at" + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.isAvailable()) {
- return putShortUnsafe(bytes, offset, val);
- } else {
- bytes[offset + 1] = (byte) val;
- val >>= 8;
- bytes[offset] = (byte) val;
- return offset + SIZEOF_SHORT;
- }
- }
-
- /**
- * Put a short value out to the specified byte array position (Unsafe).
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val short to write out
- * @return incremented offset
- */
- public static int putShortUnsafe(byte[] bytes, int offset, short val) {
- if (org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.littleEndian) {
- val = Short.reverseBytes(val);
- }
- org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.theUnsafe.putShort(bytes, (long) offset + org.apache.kylin.common.util.Bytes.LexicographicalComparerHolder.UnsafeComparer.BYTE_ARRAY_BASE_OFFSET, val);
- return offset + SIZEOF_SHORT;
- }
-
- /**
- * Put an int value as short out to the specified byte array position. Only the lower 2 bytes of
- * the short will be put into the array. The caller of the API need to make sure they will not
- * loose the value by doing so. This is useful to store an unsigned short which is represented as
- * int in other parts.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val value to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putAsShort(byte[] bytes, int offset, int val) {
- if (bytes.length - offset < SIZEOF_SHORT) {
- throw new IllegalArgumentException("Not enough room to put a short at" + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- bytes[offset + 1] = (byte) val;
- val >>= 8;
- bytes[offset] = (byte) val;
- return offset + SIZEOF_SHORT;
- }
-
- /**
- * Convert a BigDecimal value to a byte array
- *
- * @param val
- * @return the byte array
- */
- public static byte[] toBytes(BigDecimal val) {
- byte[] valueBytes = val.unscaledValue().toByteArray();
- byte[] result = new byte[valueBytes.length + SIZEOF_INT];
- int offset = putInt(result, 0, val.scale());
- putBytes(result, offset, valueBytes, 0, valueBytes.length);
- return result;
- }
-
- /**
- * Converts a byte array to a BigDecimal
- *
- * @param bytes
- * @return the char value
- */
- public static BigDecimal toBigDecimal(byte[] bytes) {
- return toBigDecimal(bytes, 0, bytes.length);
- }
-
- /**
- * Converts a byte array to a BigDecimal value
- *
- * @param bytes
- * @param offset
- * @param length
- * @return the char value
- */
- public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
- if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
- return null;
- }
-
- int scale = toInt(bytes, offset);
- byte[] tcBytes = new byte[length - SIZEOF_INT];
- System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
- return new BigDecimal(new BigInteger(tcBytes), scale);
- }
-
- /**
- * Put a BigDecimal value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val BigDecimal to write out
- * @return incremented offset
- */
- public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) {
- if (bytes == null) {
- return offset;
- }
-
- byte[] valueBytes = val.unscaledValue().toByteArray();
- byte[] result = new byte[valueBytes.length + SIZEOF_INT];
- offset = putInt(result, offset, val.scale());
- return putBytes(result, offset, valueBytes, 0, valueBytes.length);
- }
-
- /**
- * @param vint Integer to make a vint of.
- * @return Vint as bytes array.
- */
- public static byte[] vintToBytes(final long vint) {
- long i = vint;
- int size = WritableUtils.getVIntSize(i);
- byte[] result = new byte[size];
- int offset = 0;
- if (i >= -112 && i <= 127) {
- result[offset] = (byte) i;
- return result;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- result[offset++] = (byte) len;
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- result[offset++] = (byte) ((i & mask) >> shiftbits);
- }
- return result;
- }
-
- /**
- * @param buffer buffer to convert
- * @return vint bytes as an integer.
- */
- public static long bytesToVint(final byte[] buffer) {
- int offset = 0;
- byte firstByte = buffer[offset++];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset++];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * Reads a zero-compressed encoded long from input stream and returns it.
- *
- * @param buffer Binary array
- * @param offset Offset into array at which vint begins.
- * @return deserialized long from stream.
- * @throws java.io.IOException e
- */
- public static long readVLong(final byte[] buffer, final int offset) throws IOException {
- byte firstByte = buffer[offset];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset + 1 + idx];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(final byte[] left, final byte[] right) {
- return LexicographicalComparerHolder.BEST_COMPARER.compareTo(left, 0, left.length, right, 0, right.length);
- }
-
- /**
- * Lexicographically compare two arrays.
- *
- * @param buffer1 left operand
- * @param buffer2 right operand
- * @param offset1 Where to start comparing in the left buffer
- * @param offset2 Where to start comparing in the right buffer
- * @param length1 How much to compare from the left buffer
- * @param length2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) {
- return LexicographicalComparerHolder.BEST_COMPARER.compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
- }
-
- interface Comparer<T> {
- int compareTo(T buffer1, int offset1, int length1, T buffer2, int offset2, int length2);
- }
-
- static Comparer<byte[]> lexicographicalComparerJavaImpl() {
- return LexicographicalComparerHolder.PureJavaComparer.INSTANCE;
- }
-
- /**
- * Provides a lexicographical comparer implementation; either a Java
- * implementation or a faster implementation based on {@link sun.misc.Unsafe}.
- * <p/>
- * <p>Uses reflection to gracefully fall back to the Java implementation if
- * {@code Unsafe} isn't available.
- */
- static class LexicographicalComparerHolder {
- static final String UNSAFE_COMPARER_NAME = LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
-
- static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
-
- /**
- * Returns the Unsafe-using Comparer, or falls back to the pure-Java
- * implementation if unable to do so.
- */
- static Comparer<byte[]> getBestComparer() {
- try {
- Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
-
- // yes, UnsafeComparer does implement Comparer<byte[]>
- @SuppressWarnings("unchecked")
- Comparer<byte[]> comparer = (Comparer<byte[]>) theClass.getEnumConstants()[0];
- return comparer;
- } catch (Throwable t) { // ensure we really catch *everything*
- return lexicographicalComparerJavaImpl();
- }
- }
-
- enum PureJavaComparer implements Comparer<byte[]> {
- INSTANCE;
-
- @Override
- public int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) {
- // Short circuit equal case
- if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) {
- return 0;
- }
- // Bring WritableComparator code local
- int end1 = offset1 + length1;
- int end2 = offset2 + length2;
- for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
- int a = (buffer1[i] & 0xff);
- int b = (buffer2[j] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return length1 - length2;
- }
- }
-
- enum UnsafeComparer implements Comparer<byte[]> {
- INSTANCE;
-
- static final Unsafe theUnsafe;
-
- /**
- * The offset to the first element in a byte array.
- */
- static final int BYTE_ARRAY_BASE_OFFSET;
-
- static {
- theUnsafe = (Unsafe) AccessController.doPrivileged(new PrivilegedAction<Object>() {
- @Override
- public Object run() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return f.get(null);
- } catch (NoSuchFieldException e) {
- // It doesn't matter what we throw;
- // it's swallowed in getBestComparer().
- throw new Error();
- } catch (IllegalAccessException e) {
- throw new Error();
- }
- }
- });
-
- BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
-
- // sanity check - this should never fail
- if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
- throw new AssertionError();
- }
- }
-
- static final boolean littleEndian = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
-
- /**
- * Returns true if x1 is less than x2, when both values are treated as
- * unsigned long.
- */
- static boolean lessThanUnsignedLong(long x1, long x2) {
- return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE);
- }
-
- /**
- * Returns true if x1 is less than x2, when both values are treated as
- * unsigned int.
- */
- static boolean lessThanUnsignedInt(int x1, int x2) {
- return (x1 & 0xffffffffL) < (x2 & 0xffffffffL);
- }
-
- /**
- * Returns true if x1 is less than x2, when both values are treated as
- * unsigned short.
- */
- static boolean lessThanUnsignedShort(short x1, short x2) {
- return (x1 & 0xffff) < (x2 & 0xffff);
- }
-
- /**
- * Checks if Unsafe is available
- *
- * @return true, if available, false - otherwise
- */
- public static boolean isAvailable() {
- return theUnsafe != null;
- }
-
- /**
- * Lexicographically compare two arrays.
- *
- * @param buffer1 left operand
- * @param buffer2 right operand
- * @param offset1 Where to start comparing in the left buffer
- * @param offset2 Where to start comparing in the right buffer
- * @param length1 How much to compare from the left buffer
- * @param length2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- @Override
- public int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) {
-
- // Short circuit equal case
- if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) {
- return 0;
- }
- final int minLength = Math.min(length1, length2);
- final int minWords = minLength / SIZEOF_LONG;
- final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
- final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
-
- /*
- * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
- * time is no slower than comparing 4 bytes at a time even on 32-bit.
- * On the other hand, it is substantially faster on 64-bit.
- */
- for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
- long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
- long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
- long diff = lw ^ rw;
- if (littleEndian) {
- lw = Long.reverseBytes(lw);
- rw = Long.reverseBytes(rw);
- }
- if (diff != 0) {
- return lessThanUnsignedLong(lw, rw) ? -1 : 1;
- }
- }
- int offset = minWords * SIZEOF_LONG;
-
- if (minLength - offset >= SIZEOF_INT) {
- int il = theUnsafe.getInt(buffer1, offset1Adj + offset);
- int ir = theUnsafe.getInt(buffer2, offset2Adj + offset);
- if (littleEndian) {
- il = Integer.reverseBytes(il);
- ir = Integer.reverseBytes(ir);
- }
- if (il != ir) {
- return lessThanUnsignedInt(il, ir) ? -1 : 1;
- }
- offset += SIZEOF_INT;
- }
- if (minLength - offset >= SIZEOF_SHORT) {
- short sl = theUnsafe.getShort(buffer1, offset1Adj + offset);
- short sr = theUnsafe.getShort(buffer2, offset2Adj + offset);
- if (littleEndian) {
- sl = Short.reverseBytes(sl);
- sr = Short.reverseBytes(sr);
- }
- if (sl != sr) {
- return lessThanUnsignedShort(sl, sr) ? -1 : 1;
- }
- offset += SIZEOF_SHORT;
- }
- if (minLength - offset == 1) {
- int a = (buffer1[(int) (offset1 + offset)] & 0xff);
- int b = (buffer2[(int) (offset2 + offset)] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return length1 - length2;
- }
- }
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return True if equal
- */
- public static boolean equals(final byte[] left, final byte[] right) {
- // Could use Arrays.equals?
- //noinspection SimplifiableConditionalExpression
- if (left == right)
- return true;
- if (left == null || right == null)
- return false;
- if (left.length != right.length)
- return false;
- if (left.length == 0)
- return true;
-
- // Since we're often comparing adjacent sorted data,
- // it's usual to have equal arrays except for the very last byte
- // so check that first
- if (left[left.length - 1] != right[right.length - 1])
- return false;
-
- return compareTo(left, right) == 0;
- }
-
- public static boolean equals(final byte[] left, int leftOffset, int leftLen, final byte[] right, int rightOffset, int rightLen) {
- // short circuit case
- if (left == right && leftOffset == rightOffset && leftLen == rightLen) {
- return true;
- }
- // different lengths fast check
- if (leftLen != rightLen) {
- return false;
- }
- if (leftLen == 0) {
- return true;
- }
-
- // Since we're often comparing adjacent sorted data,
- // it's usual to have equal arrays except for the very last byte
- // so check that first
- if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1])
- return false;
-
- return LexicographicalComparerHolder.BEST_COMPARER.compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
- }
-
- /**
- * @param a left operand
- * @param buf right operand
- * @return True if equal
- */
- public static boolean equals(byte[] a, ByteBuffer buf) {
- if (a == null)
- return buf == null;
- if (buf == null)
- return false;
- if (a.length != buf.remaining())
- return false;
-
- // Thou shalt not modify the original byte buffer in what should be read only operations.
- ByteBuffer b = buf.duplicate();
- for (byte anA : a) {
- if (anA != b.get()) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Return true if the byte array on the right is a prefix of the byte
- * array on the left.
- */
- public static boolean startsWith(byte[] bytes, byte[] prefix) {
- return bytes != null && prefix != null && bytes.length >= prefix.length && LexicographicalComparerHolder.BEST_COMPARER.compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
- }
-
- /**
- * @param b bytes to hash
- * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what {@link org.apache.hadoop.io.Text} and
- * {@link org.apache.hadoop.hbase.io.ImmutableBytesWritable} use calculating hash code.
- */
- public static int hashCode(final byte[] b) {
- return hashCode(b, b.length);
- }
-
- /**
- * @param b value
- * @param length length of the value
- * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what {@link org.apache.hadoop.io.Text} and
- * {@link org.apache.hadoop.hbase.io.ImmutableBytesWritable} use calculating hash code.
- */
- public static int hashCode(final byte[] b, final int length) {
- return WritableComparator.hashBytes(b, length);
- }
-
- /**
- * @param b bytes to hash
- * @return A hash of <code>b</code> as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b) {
- return hashCode(b);
- }
-
- /**
- * @param b bytes to hash
- * @param length length to hash
- * @return A hash of <code>b</code> as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b, final int length) {
- return hashCode(b, length);
- }
-
- /**
- * @param a lower half
- * @param b upper half
- * @return New array that has a in lower half and b in upper half.
- */
- public static byte[] add(final byte[] a, final byte[] b) {
- return add(a, b, EMPTY_BYTE_ARRAY);
- }
-
- /**
- * @param a first third
- * @param b second third
- * @param c third third
- * @return New array made from a, b and c
- */
- public static byte[] add(final byte[] a, final byte[] b, final byte[] c) {
- byte[] result = new byte[a.length + b.length + c.length];
- System.arraycopy(a, 0, result, 0, a.length);
- System.arraycopy(b, 0, result, a.length, b.length);
- System.arraycopy(c, 0, result, a.length + b.length, c.length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to grab
- * @return First <code>length</code> bytes from <code>a</code>
- */
- public static byte[] head(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, 0, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to snarf
- * @return Last <code>length</code> bytes from <code>a</code>
- */
- public static byte[] tail(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, a.length - length, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes
- */
- public static byte[] padHead(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(padding, a);
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in <code>a</code> plus <code>length</code> appended 0 bytes
- */
- public static byte[] padTail(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(a, padding);
- }
-
- /**
- * Split passed range. Expensive operation relatively. Uses BigInteger math.
- * Useful splitting ranges for MapReduce jobs.
- *
- * @param a Beginning of range
- * @param b End of range
- * @param num Number of times to split range. Pass 1 if you want to split
- * the range in two; i.e. one split.
- * @return Array of dividing values
- */
- public static byte[][] split(final byte[] a, final byte[] b, final int num) {
- return split(a, b, false, num);
- }
-
- /**
- * Split passed range. Expensive operation relatively. Uses BigInteger math.
- * Useful splitting ranges for MapReduce jobs.
- *
- * @param a Beginning of range
- * @param b End of range
- * @param inclusive Whether the end of range is prefix-inclusive or is
- * considered an exclusive boundary. Automatic splits are generally exclusive
- * and manual splits with an explicit range utilize an inclusive end of range.
- * @param num Number of times to split range. Pass 1 if you want to split
- * the range in two; i.e. one split.
- * @return Array of dividing values
- */
- public static byte[][] split(final byte[] a, final byte[] b, boolean inclusive, final int num) {
- byte[][] ret = new byte[num + 2][];
- int i = 0;
- Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
- if (iter == null)
- return null;
- for (byte[] elem : iter) {
- ret[i++] = elem;
- }
- return ret;
- }
-
- /**
- * Iterate over keys within the passed range, splitting at an [a,b) boundary.
- */
- public static Iterable<byte[]> iterateOnSplits(final byte[] a, final byte[] b, final int num) {
- return iterateOnSplits(a, b, false, num);
- }
-
- /**
- * Iterate over keys within the passed range.
- */
- public static Iterable<byte[]> iterateOnSplits(final byte[] a, final byte[] b, boolean inclusive, final int num) {
- byte[] aPadded;
- byte[] bPadded;
- if (a.length < b.length) {
- aPadded = padTail(a, b.length - a.length);
- bPadded = b;
- } else if (b.length < a.length) {
- aPadded = a;
- bPadded = padTail(b, a.length - b.length);
- } else {
- aPadded = a;
- bPadded = b;
- }
- if (compareTo(aPadded, bPadded) >= 0) {
- throw new IllegalArgumentException("b <= a");
- }
- if (num <= 0) {
- throw new IllegalArgumentException("num cannot be <= 0");
- }
- byte[] prependHeader = { 1, 0 };
- final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
- final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
- BigInteger diffBI = stopBI.subtract(startBI);
- if (inclusive) {
- diffBI = diffBI.add(BigInteger.ONE);
- }
- final BigInteger splitsBI = BigInteger.valueOf(num + 1);
- if (diffBI.compareTo(splitsBI) < 0) {
- return null;
- }
- final BigInteger intervalBI;
- try {
- intervalBI = diffBI.divide(splitsBI);
- } catch (Exception e) {
- LOG.error("Exception caught during division", e);
- return null;
- }
-
- final Iterator<byte[]> iterator = new Iterator<byte[]>() {
- private int i = -1;
-
- @Override
- public boolean hasNext() {
- return i < num + 1;
- }
-
- @Override
- public byte[] next() {
- i++;
- if (i == 0)
- return a;
- if (i == num + 1)
- return b;
-
- BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
- byte[] padded = curBI.toByteArray();
- if (padded[1] == 0)
- padded = tail(padded, padded.length - 2);
- else
- padded = tail(padded, padded.length - 1);
- return padded;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
-
- return new Iterable<byte[]>() {
- @Override
- public Iterator<byte[]> iterator() {
- return iterator;
- }
- };
- }
-
- /**
- * @param bytes array to hash
- * @param offset offset to start from
- * @param length length to hash
- */
- public static int hashCode(byte[] bytes, int offset, int length) {
- int hash = 1;
- for (int i = offset; i < offset + length; i++)
- hash = (31 * hash) + (int) bytes[i];
- return hash;
- }
-
- /**
- * @param t operands
- * @return Array of byte arrays made from passed array of Text
- */
- public static byte[][] toByteArrays(final String[] t) {
- byte[][] result = new byte[t.length][];
- for (int i = 0; i < t.length; i++) {
- result[i] = Bytes.toBytes(t[i]);
- }
- return result;
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * <code>column</code>
- */
- public static byte[][] toByteArrays(final String column) {
- return toByteArrays(toBytes(column));
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * <code>column</code>
- */
- public static byte[][] toByteArrays(final byte[] column) {
- byte[][] result = new byte[1][];
- result[0] = column;
- return result;
- }
-
- /**
- * Binary search for keys in indexes.
- *
- * @param arr array of byte arrays to search for
- * @param key the key you want to find
- * @param offset the offset in the key you want to find
- * @param length the length of the key
- * @param comparator a comparator to compare.
- * @return zero-based index of the key, if the key is present in the array.
- * Otherwise, a value -(i + 1) such that the key is between arr[i -
- * 1] and arr[i] non-inclusively, where i is in [0, i], if we define
- * arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above
- * means that this function can return 2N + 1 different values
- * ranging from -(N + 1) to N - 1.
- */
- public static int binarySearch(byte[][] arr, byte[] key, int offset, int length, RawComparator<?> comparator) {
- int low = 0;
- int high = arr.length - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- // we have to compare in this order, because the comparator order
- // has special logic when the 'left side' is a special key.
- int cmp = comparator.compare(key, offset, length, arr[mid], 0, arr[mid].length);
- // key lives above the midpoint
- if (cmp > 0)
- low = mid + 1;
- // key lives below the midpoint
- else if (cmp < 0)
- high = mid - 1;
- // BAM. how often does this really happen?
- else
- return mid;
- }
- return -(low + 1);
- }
-
- /**
- * Bytewise binary increment/deincrement of long contained in byte array
- * on given amount.
- *
- * @param value - array of bytes containing long (length <= SIZEOF_LONG)
- * @param amount value will be incremented on (deincremented if negative)
- * @return array of bytes containing incremented long (length == SIZEOF_LONG)
- */
- public static byte[] incrementBytes(byte[] value, long amount) {
- byte[] val = value;
- if (val.length < SIZEOF_LONG) {
- // Hopefully this doesn't happen too often.
- byte[] newvalue;
- if (val[0] < 0) {
- newvalue = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1 };
- } else {
- newvalue = new byte[SIZEOF_LONG];
- }
- System.arraycopy(val, 0, newvalue, newvalue.length - val.length, val.length);
- val = newvalue;
- } else if (val.length > SIZEOF_LONG) {
- throw new IllegalArgumentException("Increment Bytes - value too big: " + val.length);
- }
- if (amount == 0)
- return val;
- if (val[0] < 0) {
- return binaryIncrementNeg(val, amount);
- }
- return binaryIncrementPos(val, amount);
- }
-
- /* increment/deincrement for positive value */
- private static byte[] binaryIncrementPos(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = value[value.length - i - 1] & 0x0ff;
- int total = val + cur;
- if (total > 255) {
- amo += sign;
- total %= 256;
- } else if (total < 0) {
- amo -= sign;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
- /* increment/deincrement for negative value */
- private static byte[] binaryIncrementNeg(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = ((~value[value.length - i - 1]) & 0x0ff) + 1;
- int total = cur - val;
- if (total >= 0) {
- amo += sign;
- } else if (total < -256) {
- amo -= sign;
- total %= 256;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
- /**
- * Writes a string as a fixed-size field, padded with zeros.
- */
- public static void writeStringFixedSize(final DataOutput out, String s, int size) throws IOException {
- byte[] b = toBytes(s);
- if (b.length > size) {
- throw new IOException("Trying to write " + b.length + " bytes (" + toStringBinary(b) + ") into a field of length " + size);
- }
-
- out.writeBytes(s);
- for (int i = 0; i < size - s.length(); ++i)
- out.writeByte(0);
- }
-
- /**
- * Reads a fixed-size field and interprets it as a string padded with zeros.
- */
- public static String readStringFixedSize(final DataInput in, int size) throws IOException {
- byte[] b = new byte[size];
- in.readFully(b);
- int n = b.length;
- while (n > 0 && b[n - 1] == 0)
- --n;
-
- return toString(b, 0, n);
- }
-
- /**
- * Copy the byte array given in parameter and return an instance
- * of a new byte array with the same length and the same content.
- *
- * @param bytes the byte array to duplicate
- * @return a copy of the given byte array
- */
- public static byte[] copy(byte[] bytes) {
- if (bytes == null)
- return null;
- byte[] result = new byte[bytes.length];
- System.arraycopy(bytes, 0, result, 0, bytes.length);
- return result;
- }
-
- /**
- * Copy the byte array given in parameter and return an instance
- * of a new byte array with the same length and the same content.
- *
- * @param bytes the byte array to copy from
- * @param offset
- * @param length
- * @return a copy of the given designated byte array
- */
- public static byte[] copy(byte[] bytes, final int offset, final int length) {
- if (bytes == null)
- return null;
- byte[] result = new byte[length];
- System.arraycopy(bytes, offset, result, 0, length);
- return result;
- }
-
- /**
- * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from
- * somewhere. (mcorgan)
- *
- * @param a Array to search. Entries must be sorted and unique.
- * @param fromIndex First index inclusive of "a" to include in the search.
- * @param toIndex Last index exclusive of "a" to include in the search.
- * @param key The byte to search for.
- * @return The index of key if found. If not found, return -(index + 1), where negative indicates
- * "not found" and the "index + 1" handles the "-0" case.
- */
- public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
- int unsignedKey = key & 0xff;
- int low = fromIndex;
- int high = toIndex - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- int midVal = a[mid] & 0xff;
-
- if (midVal < unsignedKey) {
- low = mid + 1;
- } else if (midVal > unsignedKey) {
- high = mid - 1;
- } else {
- return mid; // key found
- }
- }
- return -(low + 1); // key not found.
- }
-
- /**
- * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding
- * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes.
- *
- * @param input The byte[] to increment.
- * @return The incremented copy of "in". May be same length or 1 byte longer.
- */
- public static byte[] unsignedCopyAndIncrement(final byte[] input) {
- byte[] copy = copy(input);
- if (copy == null) {
- throw new IllegalArgumentException("cannot increment null array");
- }
- for (int i = copy.length - 1; i >= 0; --i) {
- if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum
- copy[i] = 0;
- } else {
- ++copy[i];
- return copy;
- }
- }
- // we maxed out the array
- byte[] out = new byte[copy.length + 1];
- out[0] = 1;
- System.arraycopy(copy, 0, out, 1, copy.length);
- return out;
- }
-
- public static boolean equals(List<byte[]> a, List<byte[]> b) {
- if (a == null) {
- if (b == null) {
- return true;
- }
- return false;
- }
- if (b == null) {
- return false;
- }
- if (a.size() != b.size()) {
- return false;
- }
- for (int i = 0; i < a.size(); ++i) {
- if (!Bytes.equals(a.get(i), b.get(i))) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Returns the index of the first appearance of the value {@code target} in
- * {@code array}.
- *
- * @param array an array of {@code byte} values, possibly empty
- * @param target a primitive {@code byte} value
- * @return the least index {@code i} for which {@code array[i] == target}, or
- * {@code -1} if no such index exists.
- */
- public static int indexOf(byte[] array, byte target) {
- for (int i = 0; i < array.length; i++) {
- if (array[i] == target) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Returns the start position of the first occurrence of the specified {@code
- * target} within {@code array}, or {@code -1} if there is no such occurrence.
- * <p/>
- * <p>More formally, returns the lowest index {@code i} such that {@code
- * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly
- * the same elements as {@code target}.
- *
- * @param array the array to search for the sequence {@code target}
- * @param target the array to search for as a sub-sequence of {@code array}
- */
- public static int indexOf(byte[] array, byte[] target) {
- checkNotNull(array, "array");
- checkNotNull(target, "target");
- if (target.length == 0) {
- return 0;
- }
-
- outer: for (int i = 0; i < array.length - target.length + 1; i++) {
- for (int j = 0; j < target.length; j++) {
- if (array[i + j] != target[j]) {
- continue outer;
- }
- }
- return i;
- }
- return -1;
- }
-
- /**
- * @param array an array of {@code byte} values, possibly empty
- * @param target a primitive {@code byte} value
- * @return {@code true} if {@code target} is present as an element anywhere in {@code array}.
- */
- public static boolean contains(byte[] array, byte target) {
- return indexOf(array, target) > -1;
- }
-
- /**
- * @param array an array of {@code byte} values, possibly empty
- * @param target an array of {@code byte}
- * @return {@code true} if {@code target} is present anywhere in {@code array}
- */
- public static boolean contains(byte[] array, byte[] target) {
- return indexOf(array, target) > -1;
- }
-
- /**
- * Fill given array with zeros.
- *
- * @param b array which needs to be filled with zeros
- */
- public static void zero(byte[] b) {
- zero(b, 0, b.length);
- }
-
- /**
- * Fill given array with zeros at the specified position.
- *
- * @param b
- * @param offset
- * @param length
- */
- public static void zero(byte[] b, int offset, int length) {
- checkPositionIndex(offset, b.length, "offset");
- checkArgument(length > 0, "length must be greater than 0");
- checkPositionIndex(offset + length, b.length, "offset + length");
- Arrays.fill(b, offset, offset + length, (byte) 0);
- }
-
- private static final SecureRandom RNG = new SecureRandom();
-
- /**
- * Fill given array with random bytes.
- *
- * @param b array which needs to be filled with random bytes
- */
- public static void random(byte[] b) {
- RNG.nextBytes(b);
- }
-
- /**
- * Fill given array with random bytes at the specified position.
- *
- * @param b
- * @param offset
- * @param length
- */
- public static void random(byte[] b, int offset, int length) {
- checkPositionIndex(offset, b.length, "offset");
- checkArgument(length > 0, "length must be greater than 0");
- checkPositionIndex(offset + length, b.length, "offset + length");
- byte[] buf = new byte[length];
- RNG.nextBytes(buf);
- System.arraycopy(buf, 0, b, offset, length);
- }
-
- /**
- * Create a max byte array with the specified max byte count
- *
- * @param maxByteCount the length of returned byte array
- * @return the created max byte array
- */
- public static byte[] createMaxByteArray(int maxByteCount) {
- byte[] maxByteArray = new byte[maxByteCount];
- for (int i = 0; i < maxByteArray.length; i++) {
- maxByteArray[i] = (byte) 0xff;
- }
- return maxByteArray;
- }
-
- /**
- * Create a byte array which is multiple given bytes
- *
- * @param srcBytes
- * @param multiNum
- * @return byte array
- */
- public static byte[] multiple(byte[] srcBytes, int multiNum) {
- if (multiNum <= 0) {
- return new byte[0];
- }
- byte[] result = new byte[srcBytes.length * multiNum];
- for (int i = 0; i < multiNum; i++) {
- System.arraycopy(srcBytes, 0, result, i * srcBytes.length, srcBytes.length);
- }
- return result;
- }
-
- /**
- * Convert a byte array into a hex string
- *
- * @param b
- */
- public static String toHex(byte[] b) {
- checkArgument(b.length > 0, "length must be greater than 0");
- return String.format("%x", new BigInteger(1, b));
- }
-
- /**
- * Create a byte array from a string of hash digits. The length of the
- * string must be a multiple of 2
- *
- * @param hex
- */
- public static byte[] fromHex(String hex) {
- checkArgument(hex.length() > 0, "length must be greater than 0");
- checkArgument(hex.length() % 2 == 0, "length must be a multiple of 2");
- // Make sure letters are upper case
- hex = hex.toUpperCase();
- byte[] b = new byte[hex.length() / 2];
- for (int i = 0; i < b.length; i++) {
- b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) + toBinaryFromHex((byte) hex.charAt((2 * i + 1))));
- }
- return b;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java b/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
deleted file mode 100644
index b192348..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- *
- */
-public interface BytesSerializer<T> {
-
- public static final int SERIALIZE_BUFFER_SIZE = 65536;
-
- abstract public void serialize(T value, ByteBuffer out);
-
- abstract public T deserialize(ByteBuffer in);
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java b/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
deleted file mode 100644
index ef0d827..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xjiang
- */
-public class BytesSplitter {
- private static final Logger logger = LoggerFactory.getLogger(BytesSplitter.class);
-
- private static final int[] COMMON_DELIMS = new int[] { "\177".codePointAt(0), "|".codePointAt(0), "\t".codePointAt(0), ",".codePointAt(0) };
-
- private SplittedBytes[] splitBuffers;
- private int bufferSize;
-
- public SplittedBytes[] getSplitBuffers() {
- return splitBuffers;
- }
-
- public SplittedBytes getSplitBuffer(int index) {
- return splitBuffers[index];
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public BytesSplitter(int splitLen, int bytesLen) {
- this.splitBuffers = new SplittedBytes[splitLen];
- for (int i = 0; i < splitLen; i++) {
- this.splitBuffers[i] = new SplittedBytes(bytesLen);
- }
- this.bufferSize = 0;
- }
-
- public int split(byte[] bytes, int byteLen, byte delimiter) {
- this.bufferSize = 0;
- int offset = 0;
- int length = 0;
- for (int i = 0; i < byteLen; i++) {
- if (bytes[i] == delimiter) {
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- if(length > split.value.length) {
- length = split.value.length;
- }
- System.arraycopy(bytes, offset, split.value, 0, length);
- split.length = length;
- offset = i + 1;
- length = 0;
- } else {
- length++;
- }
- }
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- if(length > split.value.length) {
- length = split.value.length;
- }
- System.arraycopy(bytes, offset, split.value, 0, length);
- split.length = length;
-
- return bufferSize;
- }
-
- public byte inferByteRowDelimiter(byte[] bytes, int byteLen, int expectedSplits) throws IOException {
-
- if (expectedSplits > this.splitBuffers.length)
- throw new IOException("expectSplits can not be greater than split buffer size");
-
- boolean delimiterFound = false;
- byte foundDelimiter = 0;
- for (int i = 0; i < bytes.length; ++i) {
- byte c = bytes[i];
- if (!Character.isLetterOrDigit((char) c)) {
- try {
- int actualSplits = this.split(bytes, byteLen, c);
- if (actualSplits == expectedSplits) {
- if (!delimiterFound) {
- logger.info("Delimiter found, value is : " + c);
- delimiterFound = true;
- foundDelimiter = c;
- } else if (c != foundDelimiter) {
- throw new IOException("Duplicate delimiter found, found delimiter is : " + foundDelimiter + " new delimiter is " + c);
- }
- }
- } catch (Exception e) {
- logger.info("Unqualified delimiter pruned, value is " + c);
- }
- }
- }
-
- if (delimiterFound)
- return foundDelimiter;
- else
- throw new IOException("No delimiter found");
- }
-
- public int detectDelim(Text value, int expectedParts) {
- for (int i = 0; i < COMMON_DELIMS.length; i++) {
- int nParts = split(value.getBytes(), value.getLength(), (byte) COMMON_DELIMS[i]);
- if (nParts == expectedParts)
- return COMMON_DELIMS[i];
- }
- throw new RuntimeException("Cannot detect delimeter from first line -- " + value.toString() + " -- expect " + expectedParts + " columns");
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for (int i = 0; i < bufferSize; i++) {
- if (i > 0)
- buf.append(", ");
-
- buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length));
- }
- return buf.toString();
- }
-
- public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) {
- List<String> splitStrings = new ArrayList<String>();
- int splitOffset = 0;
- int splitLength = 0;
- for (int i = offset; i < bytes.length; i++) {
- if (bytes[i] == delimiter) {
- String str = Bytes.toString(bytes, splitOffset, splitLength);
- splitStrings.add(str);
- splitOffset = i + 1;
- splitLength = 0;
- } else {
- splitLength++;
- }
- }
- String str = Bytes.toString(bytes, splitOffset, splitLength);
- splitStrings.add(str);
- return splitStrings;
- }
-
-}
[20/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/895739d6-27e1-4ecc-b798-5851c319ea40.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/895739d6-27e1-4ecc-b798-5851c319ea40.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/895739d6-27e1-4ecc-b798-5851c319ea40.dict
deleted file mode 100644
index c2d4216..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/895739d6-27e1-4ecc-b798-5851c319ea40.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID/30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID/30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID/30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict
deleted file mode 100644
index cab8fb3..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID/30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE/1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE/1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE/1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict
deleted file mode 100644
index 106272f..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE/1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER/bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER/bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER/bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict
deleted file mode 100644
index 7f754e0..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER/bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1/8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1/8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1/8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict
deleted file mode 100644
index 2fe839a..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1/8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3/278d7283-518a-4cd0-b6e1-2573b523bf1f.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3/278d7283-518a-4cd0-b6e1-2573b523bf1f.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3/278d7283-518a-4cd0-b6e1-2573b523bf1f.dict
deleted file mode 100644
index 44cb3ba..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3/278d7283-518a-4cd0-b6e1-2573b523bf1f.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/CAL_DT/d1094cfe-3a15-4270-b066-241c543b3fd9.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/CAL_DT/d1094cfe-3a15-4270-b066-241c543b3fd9.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/CAL_DT/d1094cfe-3a15-4270-b066-241c543b3fd9.dict
deleted file mode 100644
index 29ab100..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/CAL_DT/d1094cfe-3a15-4270-b066-241c543b3fd9.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID/f7fe5157-f9a0-4f64-ad36-d9c92b9c9e2a.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID/f7fe5157-f9a0-4f64-ad36-d9c92b9c9e2a.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID/f7fe5157-f9a0-4f64-ad36-d9c92b9c9e2a.dict
deleted file mode 100644
index 8ae8f36..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID/f7fe5157-f9a0-4f64-ad36-d9c92b9c9e2a.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME/d5c40465-75e1-40bc-a960-06308f0134a6.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME/d5c40465-75e1-40bc-a960-06308f0134a6.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME/d5c40465-75e1-40bc-a960-06308f0134a6.dict
deleted file mode 100644
index 364ba65..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME/d5c40465-75e1-40bc-a960-06308f0134a6.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID/e11524a1-0fcf-4da1-b12f-c9410610ab8d.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID/e11524a1-0fcf-4da1-b12f-c9410610ab8d.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID/e11524a1-0fcf-4da1-b12f-c9410610ab8d.dict
deleted file mode 100644
index 0fb4575..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID/e11524a1-0fcf-4da1-b12f-c9410610ab8d.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SELLER_ID/fea3a18d-3f20-4b8b-a880-7af93e69241b.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SELLER_ID/fea3a18d-3f20-4b8b-a880-7af93e69241b.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SELLER_ID/fea3a18d-3f20-4b8b-a880-7af93e69241b.dict
deleted file mode 100644
index 1e2e938..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SELLER_ID/fea3a18d-3f20-4b8b-a880-7af93e69241b.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD/fe91daba-e89b-4911-ad75-41e1335a35cb.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD/fe91daba-e89b-4911-ad75-41e1335a35cb.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD/fe91daba-e89b-4911-ad75-41e1335a35cb.dict
deleted file mode 100644
index 7b096bb..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD/fe91daba-e89b-4911-ad75-41e1335a35cb.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/TRANS_ID/7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/TRANS_ID/7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict b/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/TRANS_ID/7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict
deleted file mode 100644
index 1fa89ed..0000000
Binary files a/examples/test_case_data/localmeta/dict/DEFAULT.TEST_KYLIN_FACT/TRANS_ID/7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict
deleted file mode 100644
index 1996246..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/WEEK_BEG_DT/962b5f64-bee1-49ee-a072-af882193b719.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/WEEK_BEG_DT/962b5f64-bee1-49ee-a072-af882193b719.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/WEEK_BEG_DT/962b5f64-bee1-49ee-a072-af882193b719.dict
deleted file mode 100644
index 1185de8..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_CAL_DT/WEEK_BEG_DT/962b5f64-bee1-49ee-a072-af882193b719.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict
deleted file mode 100644
index 64dd95d..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC/bfb86010-bf4c-4534-a2b0-59d020aed197.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC/bfb86010-bf4c-4534-a2b0-59d020aed197.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC/bfb86010-bf4c-4534-a2b0-59d020aed197.dict
deleted file mode 100644
index 90bc07c..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC/bfb86010-bf4c-4534-a2b0-59d020aed197.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/CRE_USER/244af7a2-7352-4b30-811f-46e637d7a133.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/CRE_USER/244af7a2-7352-4b30-811f-46e637d7a133.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/CRE_USER/244af7a2-7352-4b30-811f-46e637d7a133.dict
deleted file mode 100644
index e41ea37..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/CRE_USER/244af7a2-7352-4b30-811f-46e637d7a133.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict
deleted file mode 100644
index 7cd811a..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_NAME/f363531d-e969-4264-bffd-ac18f8f47220.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_NAME/f363531d-e969-4264-bffd-ac18f8f47220.dict b/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_NAME/f363531d-e969-4264-bffd-ac18f8f47220.dict
deleted file mode 100644
index 91846b1..0000000
Binary files a/examples/test_case_data/localmeta/dict/EDW.TEST_SITES/SITE_NAME/f363531d-e969-4264-bffd-ac18f8f47220.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict b/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict
deleted file mode 100644
index 36da380..0000000
Binary files a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict b/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict
deleted file mode 100644
index 5c9af56..0000000
Binary files a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict b/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict
deleted file mode 100644
index 6fc3127..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/aa634e3e-22d3-4cc2-9de5-085e9ac35c1f.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict b/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict
deleted file mode 100644
index 944f478..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict
deleted file mode 100644
index a2cb760..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9bb0b83a-97b2-434f-905f-4d9e04b62018.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9bb0b83a-97b2-434f-905f-4d9e04b62018.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9bb0b83a-97b2-434f-905f-4d9e04b62018.dict
deleted file mode 100644
index 08dd48b..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9bb0b83a-97b2-434f-905f-4d9e04b62018.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict
deleted file mode 100644
index e56eefb..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict
deleted file mode 100644
index efcacde..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict
deleted file mode 100644
index 95cf51c..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/270fbfb0-281c-4602-8413-2970a7439c47.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/270fbfb0-281c-4602-8413-2970a7439c47.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/270fbfb0-281c-4602-8413-2970a7439c47.dict
deleted file mode 100644
index c3caf94..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/270fbfb0-281c-4602-8413-2970a7439c47.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict
deleted file mode 100644
index 823c00d..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict
deleted file mode 100644
index 5236f78..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict
deleted file mode 100644
index bc1d8e8..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict
deleted file mode 100644
index 0bbe66f..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict
deleted file mode 100644
index 1d25663..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict
deleted file mode 100644
index a046dbb..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict
deleted file mode 100644
index c1a4196..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict
deleted file mode 100644
index b60e34f..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict
deleted file mode 100644
index 2c47f75..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict
deleted file mode 100644
index c72d118..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict
deleted file mode 100644
index 4fd22f8..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/70c8d6ef-f55b-4cdd-8be4-9c2b236cc8e9.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/70c8d6ef-f55b-4cdd-8be4-9c2b236cc8e9.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/70c8d6ef-f55b-4cdd-8be4-9c2b236cc8e9.dict
deleted file mode 100644
index 4e7a8cc..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/70c8d6ef-f55b-4cdd-8be4-9c2b236cc8e9.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict
deleted file mode 100644
index 67dbea4..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/ac797050-e152-4923-a20a-02282fcf6086.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/ac797050-e152-4923-a20a-02282fcf6086.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/ac797050-e152-4923-a20a-02282fcf6086.dict
deleted file mode 100644
index a786b20..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/ac797050-e152-4923-a20a-02282fcf6086.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict
deleted file mode 100644
index df0c657..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict
deleted file mode 100644
index 3a28967..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict
deleted file mode 100644
index e898c45..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/CAL_DT/48433f91-0d68-495f-b7f2-295414591275.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/CAL_DT/48433f91-0d68-495f-b7f2-295414591275.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/CAL_DT/48433f91-0d68-495f-b7f2-295414591275.dict
deleted file mode 100644
index 557c66a..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/CAL_DT/48433f91-0d68-495f-b7f2-295414591275.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/ITEM_COUNT/73c9bfe1-6496-4ff8-9467-6cbee2924c16.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/ITEM_COUNT/73c9bfe1-6496-4ff8-9467-6cbee2924c16.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/ITEM_COUNT/73c9bfe1-6496-4ff8-9467-6cbee2924c16.dict
deleted file mode 100644
index 4892e3a..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/ITEM_COUNT/73c9bfe1-6496-4ff8-9467-6cbee2924c16.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/4059cab1-9b99-47ed-a494-751da52a7d58.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/4059cab1-9b99-47ed-a494-751da52a7d58.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/4059cab1-9b99-47ed-a494-751da52a7d58.dict
deleted file mode 100644
index dc1bfe9..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/4059cab1-9b99-47ed-a494-751da52a7d58.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/96b7c577-b209-45b3-a848-4d2d7af5c0cc.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/96b7c577-b209-45b3-a848-4d2d7af5c0cc.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/96b7c577-b209-45b3-a848-4d2d7af5c0cc.dict
deleted file mode 100644
index 209f3ed..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/96b7c577-b209-45b3-a848-4d2d7af5c0cc.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/9fc2360e-172a-43f3-acef-be16748a9bb7.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/9fc2360e-172a-43f3-acef-be16748a9bb7.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/9fc2360e-172a-43f3-acef-be16748a9bb7.dict
deleted file mode 100644
index 35e5639..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LEAF_CATEG_ID/9fc2360e-172a-43f3-acef-be16748a9bb7.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_FORMAT_NAME/bd9f6b22-36ba-4e6b-92aa-0d585faf0b39.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_FORMAT_NAME/bd9f6b22-36ba-4e6b-92aa-0d585faf0b39.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_FORMAT_NAME/bd9f6b22-36ba-4e6b-92aa-0d585faf0b39.dict
deleted file mode 100644
index 0da7747..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_FORMAT_NAME/bd9f6b22-36ba-4e6b-92aa-0d585faf0b39.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_SITE_ID/7df5789b-0280-453c-b406-b75cad6770d1.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_SITE_ID/7df5789b-0280-453c-b406-b75cad6770d1.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_SITE_ID/7df5789b-0280-453c-b406-b75cad6770d1.dict
deleted file mode 100644
index 05afc58..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/LSTG_SITE_ID/7df5789b-0280-453c-b406-b75cad6770d1.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/PRICE/927dde3f-6999-4434-b57c-adfa73160334.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/PRICE/927dde3f-6999-4434-b57c-adfa73160334.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/PRICE/927dde3f-6999-4434-b57c-adfa73160334.dict
deleted file mode 100644
index 90c2876..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/PRICE/927dde3f-6999-4434-b57c-adfa73160334.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SELLER_ID/6e285e1e-ed16-4012-9f1e-f950dd6927ce.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SELLER_ID/6e285e1e-ed16-4012-9f1e-f950dd6927ce.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SELLER_ID/6e285e1e-ed16-4012-9f1e-f950dd6927ce.dict
deleted file mode 100644
index 829c33c..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SELLER_ID/6e285e1e-ed16-4012-9f1e-f950dd6927ce.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SLR_SEGMENT_CD/8300bf83-053e-48bb-8c87-88c8d483afd1.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SLR_SEGMENT_CD/8300bf83-053e-48bb-8c87-88c8d483afd1.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SLR_SEGMENT_CD/8300bf83-053e-48bb-8c87-88c8d483afd1.dict
deleted file mode 100644
index a676d11..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/SLR_SEGMENT_CD/8300bf83-053e-48bb-8c87-88c8d483afd1.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/TRANS_ID/c8d19f95-b6cd-4219-a114-54aaddcb2909.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/TRANS_ID/c8d19f95-b6cd-4219-a114-54aaddcb2909.dict b/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/TRANS_ID/c8d19f95-b6cd-4219-a114-54aaddcb2909.dict
deleted file mode 100644
index 12637ad..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_KYLIN_FACT/TRANS_ID/c8d19f95-b6cd-4219-a114-54aaddcb2909.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict
deleted file mode 100644
index afe548d..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict
deleted file mode 100644
index 460da2b..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict
deleted file mode 100644
index 7a97f71..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict
deleted file mode 100644
index c6a0587..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict
deleted file mode 100644
index 6f270fe..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict
deleted file mode 100644
index 5be36a1..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict
deleted file mode 100644
index 87057cd..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict
deleted file mode 100644
index 194b7e2..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict
deleted file mode 100644
index 0a60fc3..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict
deleted file mode 100644
index eb64935..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict
deleted file mode 100644
index a10c74b..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict
deleted file mode 100644
index d3530cf..0000000
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
deleted file mode 100644
index e38e55b..0000000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
- "name": "test_kylin_hybrid_inner_join",
- "realizations": [
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- }
- ],
- "cost": 40,
- "last_modified": 1420016227424,
- "create_time": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
deleted file mode 100644
index 48cebe6..0000000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
- "name": "test_kylin_hybrid_left_join",
- "realizations": [
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_left_join_empty"
- }
- ],
- "cost": 40,
- "last_modified": 1420016227424,
- "create_time": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
deleted file mode 100644
index ea60ae4..0000000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
- "name": "test_kylin_hybrid_ready",
- "realizations": [
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_ready_2_segments"
- },
- {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_ready"
- }
- ],
- "cost": 40,
- "last_modified": 1420016227424,
- "create_time": null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/invertedindex/test_kylin_ii.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii.json
deleted file mode 100644
index 9b0d478..0000000
--- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
- "uuid": "daa53e80-41be-49a5-89ca-9fb7294db186",
- "name": "test_kylin_ii",
- "owner": null,
- "version": null,
- "cost": 10,
- "status": "READY",
- "segments": [
- {
- "uuid": null,
- "name": "19700101000000_20140901000000",
- "status": "READY",
- "dictionaries": {
- "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID": "/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/SITE_ID/30c9d5f0-abe4-4d1c-a147-610234d90ff1.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/9355165b-06ad-4c04-977c-a251e66e7e98.dict",
- "DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME": "/dict/DEFAULT.TEST_KYLIN_FACT/LSTG_FORMAT_NAME/d5c40465-75e1-40bc-a960-06308f0134a6.dict",
- "DEFAULT.TEST_KYLIN_FACT/SELLER_ID": "/dict/DEFAULT.TEST_KYLIN_FACT/SELLER_ID/fea3a18d-3f20-4b8b-a880-7af93e69241b.dict",
- "EDW.TEST_SITES/CRE_USER": "/dict/EDW.TEST_SITES/CRE_USER/244af7a2-7352-4b30-811f-46e637d7a133.dict",
- "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict",
- "EDW.TEST_CAL_DT/CAL_DT": "/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict",
- "DEFAULT.TEST_KYLIN_FACT/TRANS_ID": "/dict/DEFAULT.TEST_KYLIN_FACT/TRANS_ID/7fb8fed8-7f3b-4089-a85d-3ac07f575c82.dict",
- "EDW.TEST_CAL_DT/WEEK_BEG_DT": "/dict/EDW.TEST_CAL_DT/WEEK_BEG_DT/962b5f64-bee1-49ee-a072-af882193b719.dict",
- "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD": "/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict",
- "EDW.TEST_SITES/SITE_NAME": "/dict/EDW.TEST_SITES/SITE_NAME/f363531d-e969-4264-bffd-ac18f8f47220.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_DATE/1fc93a94-1feb-4af4-8078-81a6f1b65e2b.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/895739d6-27e1-4ecc-b798-5851c319ea40.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/38361fbc-b875-4273-b8b4-1b8a26ef8570.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD": "/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/bec11fda-9ae0-4668-98ea-f0f4e9dd6993.dict",
- "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC": "/dict/EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_DESC/bfb86010-bf4c-4534-a2b0-59d020aed197.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD1/8ae44fb8-b01a-4db1-a901-dc5f463038cb.dict",
- "EDW.TEST_SITES/SITE_ID": "/dict/EDW.TEST_SITES/SITE_ID/4ef43390-b07e-4d4c-872a-77c0bd783acb.dict",
- "DEFAULT.TEST_KYLIN_FACT/CAL_DT": "/dict/EDW.TEST_CAL_DT/CAL_DT/5e4b4f35-0fc8-4940-b123-b18c9f77da19.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/USER_DEFINED_FIELD3/278d7283-518a-4cd0-b6e1-2573b523bf1f.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/UPD_USER/bb1bb7a5-b02d-45eb-b3c8-f1a4515264ca.dict",
- "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME": "/dict/DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/b2d6fae1-eaac-4ac2-8a01-42e5c8b5c198.dict"
- },
- "storage_location_identifier": "test_III",
- "date_range_start": 0,
- "date_range_end": 0,
- "size_kb": 0,
- "input_records": 0,
- "input_records_size": 0,
- "last_build_time": 0,
- "last_build_job_id": null,
- "create_time": null,
- "binary_signature": null
- }
- ],
- "last_modified": 1420016227424,
- "descriptor": "test_kylin_ii_desc",
- "create_time": null,
- "input_records_size": 0,
- "size_kb": 0,
- "source_records_count": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
deleted file mode 100644
index a5ee5e5..0000000
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_desc.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
- "uuid": "74bf87b5-c7b5-4420-a12a-07f6b37b3187",
- "last_modified": 0,
- "name": "test_kylin_ii_desc",
- "fact_table": "default.test_kylin_fact",
- "model_name": "test_kylin_ii_model_desc",
- "timestamp_dimension": "cal_dt",
- "value_dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- },
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT"
- ],
- "sharding": 4,
- "slice_size": 50000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
deleted file mode 100644
index ddff900..0000000
--- a/examples/test_case_data/localmeta/kylin.properties
+++ /dev/null
@@ -1,77 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=localhost:7070
-
-# The metadata store in hbase
-kylin.metadata.url=
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_job_conf.xml b/examples/test_case_data/localmeta/kylin_job_conf.xml
deleted file mode 100644
index 81704f2..0000000
--- a/examples/test_case_data/localmeta/kylin_job_conf.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json
deleted file mode 100644
index 4165df3..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "name": "test_kylin_ii_model_desc",
-
- "fact_table": "DEFAULT.TEST_KYLIN_FACT",
- "lookups": [
- {
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join": {
- "type": "left",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SITES",
- "join": {
- "type": "left",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "join": {
- "type": "left",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json
deleted file mode 100644
index f5a6a85..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
- "uuid" : "9c0f4ee2-1ccb-4b07-a38e-4c298563e0f7",
- "name" : "test_kylin_with_slr_left_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "left",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "left",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "left",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- }
- } ],
- "capacity" : "MEDIUM",
- "last_modified" : 1422435345352,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json
deleted file mode 100644
index 279b002..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "uuid" : "ff527b94-f860-44c3-8452-93b17774c647",
- "name" : "test_kylin_with_slr_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- }
- } ],
- "capacity" : "MEDIUM",
- "last_modified" : 1422435345324,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json
deleted file mode 100644
index 4baebaf..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "uuid" : "9a4207bd-3bf9-48dd-b334-cf27fa55d629",
- "name" : "test_kylin_without_slr_left_join_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "left",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "left",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "left",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- }
- } ],
- "capacity" : "MEDIUM",
- "last_modified" : 1422435345372,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json
deleted file mode 100644
index edf2d29..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "name" : "test_kylin_without_slr_model_desc",
- "lookups" : [ {
- "table" : "EDW.TEST_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "CAL_DT" ]
- }
- }, {
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SITES",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SITE_ID" ],
- "foreign_key" : [ "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "SELLER_TYPE_CD" ],
- "foreign_key" : [ "SLR_SEGMENT_CD" ]
- }
- } ],
- "capacity" : "SMALL",
- "last_modified" : 1422435345362,
- "fact_table" : "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : null,
- "partition_date_start" : 0,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
deleted file mode 100644
index 639f3be..0000000
--- a/examples/test_case_data/localmeta/project/default.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "uuid": "1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b",
- "name": "default",
- "realizations": [
- {
- "name": "test_kylin_with_slr_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_empty"
- },
- {
- "name": "test_kylin_without_slr_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- },
- {
- "name": "test_kylin_with_slr_left_join_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- {
- "name": "test_kylin_without_slr_left_join_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_left_join_empty"
- },
- {
- "name": "test_kylin_ii",
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii"
- },
- {
- "name": "test_kylin_hybrid_inner_join",
- "type": "HYBRID",
- "realization": "test_kylin_hybrid_inner_join"
- },
- {
- "name": "test_kylin_hybrid_left_join",
- "type": "HYBRID",
- "realization": "test_kylin_hybrid_left_join"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/project/onlyinner.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/onlyinner.json b/examples/test_case_data/localmeta/project/onlyinner.json
deleted file mode 100644
index a3ab387..0000000
--- a/examples/test_case_data/localmeta/project/onlyinner.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c92b",
- "name": "onlyinner",
- "realizations": [
- {
- "name": "test_kylin_with_slr_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_empty"
- },
- {
- "name": "test_kylin_without_slr_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/project/onlyleft.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/onlyleft.json b/examples/test_case_data/localmeta/project/onlyleft.json
deleted file mode 100644
index 014ff4b..0000000
--- a/examples/test_case_data/localmeta/project/onlyleft.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-"uuid" : "1eaca32a-a33e-4b69-83dd-0bb8b1f8c93b",
- "name": "onlyleft",
- "realizations": [
- {
- "name": "test_kylin_with_slr_left_join_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- {
- "name": "test_kylin_without_slr_left_join_model_desc",
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_left_join_empty"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
deleted file mode 100644
index 7bcd092..0000000
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
+++ /dev/null
@@ -1,151 +0,0 @@
-{
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
- "name" : "TEST_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
deleted file mode 100644
index c0dff16..0000000
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
- "name" : "TEST_KYLIN_FACT",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal(19,4)"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "int"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
[43/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
deleted file mode 100644
index 03da261..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Writable;
-
-public class BytesUtil {
-
- public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
-
- public static void writeLong(long num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static long readLong(byte[] bytes, int offset, int size) {
- long integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (long) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static int readUnsigned(byte[] bytes, int offset, int size) {
- int integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static void writeSigned(int num, byte[] bytes, int offset, int size) {
- writeUnsigned(num, bytes, offset, size);
- }
-
- public static int readSigned(byte[] bytes, int offset, int size) {
- int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- /**
- * No. bytes needed to store a value as big as the given
- */
- public static int sizeForValue(int maxValue) {
- int size = 0;
- while (maxValue > 0) {
- size++;
- maxValue >>>= 8;
- }
- return size;
- }
-
- public static int compareByteUnsigned(byte b1, byte b2) {
- int i1 = (int) b1 & 0xFF;
- int i2 = (int) b2 & 0xFF;
- return i1 - i2;
- }
-
- public static byte[] subarray(byte[] bytes, int start, int end) {
- byte[] r = new byte[end - start];
- System.arraycopy(bytes, start, r, 0, r.length);
- return r;
- }
-
- public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
- int r = 0;
- for (int i = 0; i < length; i++) {
- r = src[srcOffset + i] - dst[dstOffset + i];
- if (r != 0)
- break;
- }
- return r;
- }
-
- // from WritableUtils
- // ============================================================================
-
- public static void writeVInt(int i, ByteBuffer out) {
- writeVLong(i, out);
- }
-
- public static void writeVLong(long i, ByteBuffer out) {
- if (i >= -112 && i <= 127) {
- out.put((byte) i);
- return;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- out.put((byte) len);
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- out.put((byte) ((i & mask) >> shiftbits));
- }
- }
-
- public static long readVLong(ByteBuffer in) {
- byte firstByte = in.get();
- int len = decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = in.get();
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
- }
-
- public static int readVInt(ByteBuffer in) {
- long n = readVLong(in);
- if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
- throw new IllegalArgumentException("value too long to fit in integer");
- }
- return (int) n;
- }
-
- private static boolean isNegativeVInt(byte value) {
- return value < -120 || (value >= -112 && value < 0);
- }
-
- private static int decodeVIntSize(byte value) {
- if (value >= -112) {
- return 1;
- } else if (value < -120) {
- return -119 - value;
- }
- return -111 - value;
- }
-
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeLong(long num, ByteBuffer out) {
- for (int i = 0; i < 8; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static long readLong(ByteBuffer in) {
- long integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < 8; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeUTFString(String str, ByteBuffer out) {
- byte[] bytes = str == null ? null : Bytes.toBytes(str);
- writeByteArray(bytes, out);
- }
-
- public static String readUTFString(ByteBuffer in) {
- byte[] bytes = readByteArray(in);
- return bytes == null ? null : Bytes.toString(bytes);
- }
-
-
-
- public static void writeAsciiString(String str, ByteBuffer out) {
- if (str == null) {
- BytesUtil.writeVInt(-1, out);
- return;
- }
- int len = str.length();
- BytesUtil.writeVInt(len, out);
- for (int i = 0; i < len; i++) {
- out.put((byte) str.charAt(i));
- }
- }
-
- public static String readAsciiString(ByteBuffer in) {
- int len = BytesUtil.readVInt(in);
- if (len < 0) {
- return null;
- }
- String result;
- try {
- if (in.hasArray()) {
- int pos = in.position();
- result = new String(in.array(), pos, len, "ISO-8859-1");
- in.position(pos + len);
- } else {
- byte[] tmp = new byte[len];
- in.get(tmp);
- result = new String(tmp, "ISO-8859-1");
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return result;
- }
-
- public static void writeAsciiStringArray(String[] strs, ByteBuffer out) {
- writeVInt(strs.length, out);
- for (int i = 0; i < strs.length; i++)
- writeAsciiString(strs[i], out);
- }
-
- public static String[] readAsciiStringArray(ByteBuffer in) {
- int len = readVInt(in);
- String[] strs = new String[len];
- for (int i = 0; i < len; i++)
- strs[i] = readAsciiString(in);
- return strs;
- }
-
- public static void writeIntArray(int[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- for (int i = 0; i < array.length; ++i) {
- writeVInt(array[i], out);
- }
- }
-
- public static int[] readIntArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
- int[] array = new int[len];
-
- for (int i = 0; i < len; ++i) {
- array[i] = readVInt(in);
- }
- return array;
- }
-
- public static void writeByteArray(byte[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- out.put(array);
- }
-
- public static byte[] readByteArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
-
- byte[] array = new byte[len];
- in.get(array);
- return array;
- }
-
-
- public static int peekByteArrayLength(ByteBuffer in) {
- int start = in.position();
- int arrayLen = readVInt(in);
- int sizeLen = in.position() - start;
- in.position(start);
-
- if (arrayLen < 0)
- return sizeLen;
- else
- return sizeLen + arrayLen;
- }
-
-
- public static void writeBooleanArray(boolean[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- byte b_true = (byte) 1;
- byte b_false = (byte) 0;
- for (int i = 0; i < array.length; i++) {
- if (array[i])
- out.put(b_true);
- else
- out.put(b_false);
- }
- }
-
- public static boolean[] readBooleanArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
-
- boolean[] array = new boolean[len];
- byte b_true = (byte) 1;
- for (int i = 0; i < array.length; i++) {
- byte temp = in.get();
- if (temp == b_true)
- array[i] = true;
- else
- array[i] = false;
- }
- return array;
- }
-
- public static byte[] toBytes(Writable writable) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bout);
- writable.write(out);
- out.close();
- bout.close();
- return bout.toByteArray();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static String toReadableText(byte[] array) {
- return toHex(array);
- }
-
- /**
- * this method only works for hex strings
- */
- public static byte[] fromReadableText(String text) {
- String[] tokens = text.split("\\\\x");
- byte[] ret = new byte[tokens.length - 1];
- for (int i = 1; i < tokens.length; ++i) {
- int x = Bytes.toBinaryFromHex((byte) tokens[i].charAt(0));
- x = x << 4;
- int y = Bytes.toBinaryFromHex((byte) tokens[i].charAt(1));
- ret[i - 1] = (byte) (x + y);
- }
- return ret;
- }
-
- public static String toHex(byte[] array) {
- return toHex(new ImmutableBytesWritable(array));
- }
-
- public static String toHex(ImmutableBytesWritable bytes) {
- byte[] array = bytes.get();
- int offset = bytes.getOffset();
- int length = bytes.getLength();
- StringBuilder sb = new StringBuilder(length * 4);
- for (int i = 0; i < length; i++) {
- int b = array[offset + i];
- sb.append(String.format("\\x%02X", b & 0xFF));
- }
- return sb.toString();
- }
-
- public static void main(String[] args) throws Exception {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java b/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
deleted file mode 100644
index 23e1973..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/CaseInsensitiveStringMap.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Created by qianzhou on 12/3/14.
- */
-public class CaseInsensitiveStringMap<T> implements Map<String, T> {
-
- private Map<String, T> innerMap;
-
- public CaseInsensitiveStringMap() {
- this(new HashMap<String, T>());
- }
-
- public CaseInsensitiveStringMap(Map<String, T> innerMap) {
- this.innerMap = innerMap;
- }
-
- @Override
- public int size() {
- return innerMap.size();
- }
-
- @Override
- public boolean isEmpty() {
- return innerMap.isEmpty();
- }
-
- @Override
- public boolean containsKey(Object key) {
- return key != null ? innerMap.containsKey(key.toString().toUpperCase()) : false;
- }
-
- @Override
- public boolean containsValue(Object value) {
- return value != null ? innerMap.containsValue(value) : false;
- }
-
- @Override
- public T get(Object key) {
- return key != null ? innerMap.get(key.toString().toUpperCase()) : null;
- }
-
- @Override
- public T put(String key, T value) {
- return key != null ? innerMap.put(key.toString().toUpperCase(), value) : null;
- }
-
- @Override
- public T remove(Object key) {
- return key != null ? innerMap.remove(key.toString().toUpperCase()) : null;
- }
-
- @Override
- public void putAll(Map<? extends String, ? extends T> m) {
- innerMap.putAll(m);
- }
-
- @Override
- public void clear() {
- innerMap.clear();
- }
-
- @Override
- public Set<String> keySet() {
- return innerMap.keySet();
- }
-
- @Override
- public Collection<T> values() {
- return innerMap.values();
- }
-
- @Override
- public Set<Entry<String, T>> entrySet() {
- return innerMap.entrySet();
- }
-
- @Override
- public String toString() {
- return this.innerMap.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
deleted file mode 100644
index 7c8cda7..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.net.URL;
-import java.net.URLClassLoader;
-
-/**
- * @author xduo
- *
- */
-public class ClassUtil {
-
- public static void addClasspath(String path) throws Exception {
- System.out.println("Adding path " + path + " to class path");
- File file = new File(path);
-
- if (file.exists()) {
- URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
- Class<URLClassLoader> urlClass = URLClassLoader.class;
- Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
- method.setAccessible(true);
- method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
- }
- }
-
- @SuppressWarnings("unchecked")
- public static <T> Class<? extends T> forName(String name, Class<T> clz) throws ClassNotFoundException {
- if (name.startsWith("com.kylinolap")) {
- name = "org.apache.kylin" + name.substring("com.kylinolap".length());
- }
- return (Class<? extends T>) Class.forName(name);
- }
-
- public static Object newInstance(String clz) {
- try {
- return forName(clz, Object.class).newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- public static void updateFinalField(Class<?> clz, String fieldName, Object obj, Object newValue) {
- try {
- Field field = clz.getDeclaredField(fieldName);
- field.setAccessible(true);
-
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- field.set(obj, newValue);
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
deleted file mode 100644
index 83fb6e0..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.apache.commons.io.FileUtils;
-
-/**
- * @author yangli9
- */
-public class CliCommandExecutor {
-
- private String remoteHost;
- private String remoteUser;
- private String remotePwd;
- private int remoteTimeoutSeconds = 3600;
-
- public CliCommandExecutor() {
- }
-
- public void setRunAtRemote(String host, String user, String pwd) {
- this.remoteHost = host;
- this.remoteUser = user;
- this.remotePwd = pwd;
- }
-
- public void setRunAtLocal() {
- this.remoteHost = null;
- this.remoteUser = null;
- this.remotePwd = null;
- }
-
- public void copyFile(String localFile, String destDir) throws IOException {
- if (remoteHost == null)
- copyNative(localFile, destDir);
- else
- copyRemote(localFile, destDir);
- }
-
- private void copyNative(String localFile, String destDir) throws IOException {
- File src = new File(localFile);
- File dest = new File(destDir, src.getName());
- FileUtils.copyFile(src, dest);
- }
-
- private void copyRemote(String localFile, String destDir) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd);
- try {
- ssh.scpFileToRemote(localFile, destDir);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- public Pair<Integer, String> execute(String command) throws IOException {
- return execute(command, new SoutLogger());
- }
-
- public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException {
- Pair<Integer, String> r;
- if (remoteHost == null) {
- r = runNativeCommand(command, logAppender);
- } else {
- r = runRemoteCommand(command, logAppender);
- }
-
- if (r.getFirst() != 0)
- throw new IOException("OS command error exit with " + r.getFirst() //
- + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") //
- + " -- " + command + "\n" + r.getSecond());
-
- return r;
- }
-
- private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd);
-
- SSHClientOutput sshOutput;
- try {
- sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender);
- int exitCode = sshOutput.getExitCode();
- String output = sshOutput.getText();
- return new Pair<Integer, String>(exitCode, output);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException {
- String[] cmd = new String[3];
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Windows")) {
- cmd[0] = "cmd.exe";
- cmd[1] = "/C";
- } else {
- cmd[0] = "/bin/bash";
- cmd[1] = "-c";
- }
- cmd[2] = command;
-
- ProcessBuilder builder = new ProcessBuilder(cmd);
- builder.redirectErrorStream(true);
- Process proc = builder.start();
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
- String line;
- StringBuilder result = new StringBuilder();
- while ((line = reader.readLine()) != null) {
- result.append(line).append('\n');
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
-
- try {
- int exitCode = proc.waitFor();
- return new Pair<Integer, String>(exitCode, result.toString());
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
deleted file mode 100644
index 69fc1a2..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class DateFormat {
-
- public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
- public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
- public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
-
- static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
-
- public static SimpleDateFormat getDateFormat(String datePattern) {
- ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
- if (formatThreadLocal == null) {
- threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
- }
- SimpleDateFormat format = formatThreadLocal.get();
- if (format == null) {
- format = new SimpleDateFormat(datePattern);
- format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
- formatThreadLocal.set(format);
- }
- return format;
- }
-
- public static String formatToDateStr(long millis) {
- return formatToDateStr(millis, DEFAULT_DATE_PATTERN);
- }
-
- public static String formatToDateStr(long millis, String pattern) {
- return getDateFormat(pattern).format(new Date(millis));
- }
-
- public static String formatToTimeStr(long millis) {
- return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis));
- }
-
- public static String dateToString(Date date) {
- return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- }
-
- public static String dateToString(Date date, String pattern) {
- return getDateFormat(pattern).format(date);
- }
-
- public static Date stringToDate(String str) {
- return stringToDate(str, DEFAULT_DATE_PATTERN);
- }
-
- public static Date stringToDate(String str, String pattern) {
- Date date = null;
- try {
- date = getDateFormat(pattern).parse(str);
- } catch (ParseException e) {
- throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
- }
- return date;
- }
-
- public static long stringToMillis(String str) {
- if (isAllDigits(str)) {
- return Long.parseLong(str);
- } else if (str.length() == 10) {
- return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
- } else if (str.length() == 19) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
- } else if (str.length() == 23) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
- } else {
- throw new IllegalArgumentException("there is no valid date pattern for:" + str);
- }
- }
-
- private static boolean isAllDigits(String str) {
- for (int i = 0, n = str.length(); i < n; i++) {
- if (Character.isDigit(str.charAt(i)) == false)
- return false;
- }
- return true;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/common/src/main/java/org/apache/kylin/common/util/Dictionary.java
deleted file mode 100644
index 0168609..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * A bi-way dictionary that maps from dimension/column values to IDs and vice
- * versa. By storing IDs instead of real values, the size of cube is
- * significantly reduced.
- *
- * - IDs are smallest integers possible for the cardinality of a column, for the
- * purpose of minimal storage space - IDs preserve ordering of values, such that
- * range query can be applied to IDs directly
- *
- * A dictionary once built, is immutable. This allows optimal memory footprint
- * by e.g. flatten the Trie structure into a byte array, replacing node pointers
- * with array offsets.
- *
- * @author yangli9
- */
-abstract public class Dictionary<T> implements Writable {
-
- public static final byte NULL = (byte) 0xff;
-
- // ID with all bit-1 (0xff e.g.) reserved for NULL value
- public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff };
-
- abstract public int getMinId();
-
- abstract public int getMaxId();
-
- public int getSize() {
- return getMaxId() - getMinId() + 1;
- }
-
- /**
- * @return the size of an ID in bytes, determined by the cardinality of
- * column
- */
- abstract public int getSizeOfId();
-
- /**
- * @return the (maximum) size of value in bytes, determined by the longest
- * value of column
- */
- abstract public int getSizeOfValue();
-
- /**
- * Convenient form of <code>getIdFromValue(value, 0)</code>
- */
- final public int getIdFromValue(T value) throws IllegalArgumentException {
- return getIdFromValue(value, 0);
- }
-
- /**
- * Returns the ID integer of given value. In case of not found
- * <p>
- * - if roundingFlag=0, throw IllegalArgumentException; <br>
- * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
- * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
- * <p>
- * The implementation often has cache, thus faster than the byte[] version getIdFromValueBytes()
- *
- * @throws IllegalArgumentException
- * if value is not found in dictionary and rounding is off or
- * failed
- */
- final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
- if (isNullObjectForm(value))
- return nullId();
- else
- return getIdFromValueImpl(value, roundingFlag);
- }
-
- protected boolean isNullObjectForm(T value) {
- return value == null;
- }
-
- abstract protected int getIdFromValueImpl(T value, int roundingFlag);
-
- /**
- * @return the value corresponds to the given ID
- * @throws IllegalArgumentException
- * if ID is not found in dictionary
- */
- final public T getValueFromId(int id) {
- if (isNullId(id))
- return null;
- else
- return getValueFromIdImpl(id);
- }
-
- abstract protected T getValueFromIdImpl(int id);
-
- /**
- * Convenient form of
- * <code>getIdFromValueBytes(value, offset, len, 0)</code>
- */
- final public int getIdFromValueBytes(byte[] value, int offset, int len) {
- return getIdFromValueBytes(value, offset, len, 0);
- }
-
- /**
- * A lower level API, return ID integer from raw value bytes. In case of not found
- * <p>
- * - if roundingFlag=0, throw IllegalArgumentException; <br>
- * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
- * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
- * <p>
- * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
- *
- * @throws IllegalArgumentException
- * if value is not found in dictionary and rounding is off or failed
- */
- final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) {
- if (isNullByteForm(value, offset, len))
- return nullId();
- else
- return getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- }
-
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null;
- }
-
- abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
-
- /**
- * A lower level API, get byte values from ID, return the number of bytes
- * written. Bypassing the cache layer, this could be significantly slower
- * than getIdFromValue(T value).
- *
- * @return size of value bytes, 0 if empty string, -1 if null
- *
- * @throws IllegalArgumentException
- * if ID is not found in dictionary
- */
- final public int getValueBytesFromId(int id, byte[] returnValue, int offset) {
- if (isNullId(id))
- return -1;
- else
- return getValueBytesFromIdImpl(id, returnValue, offset);
- }
-
- abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
-
- abstract public void dump(PrintStream out);
-
- public int nullId() {
- return NULL_ID[getSizeOfId()];
- }
-
- public boolean isNullId(int id) {
- int nullId = NULL_ID[getSizeOfId()];
- return (nullId & id) == nullId;
- }
-
- /** utility that converts a dictionary ID to string, preserving order */
- public static String dictIdToString(byte[] idBytes, int offset, int length) {
- try {
- return new String(idBytes, offset, length, "ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- // never happen
- return null;
- }
- }
-
- /** the reverse of dictIdToString(), returns integer ID */
- public static int stringToDictId(String str) {
- try {
- byte[] bytes = str.getBytes("ISO-8859-1");
- return BytesUtil.readUnsigned(bytes, 0, bytes.length);
- } catch (UnsupportedEncodingException e) {
- // never happen
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java b/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
deleted file mode 100644
index 093ac9e..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/HBaseRegionSizeCalculator.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** This class will come with HBase 2.0 in package org.apache.hadoop.hbase.util **/
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HBaseRegionSizeCalculator {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseRegionSizeCalculator.class);
-
- /**
- * Maps each region to its size in bytes.
- **/
- private final Map<byte[], Long> sizeMap = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
-
- static final String ENABLE_REGIONSIZECALCULATOR = "hbase.regionsizecalculator.enable";
-
- /**
- * Computes size of each region for table and given column families.
- * */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
-
- try {
- if (!enabled(table.getConfiguration())) {
- logger.info("Region size calculation disabled.");
- return;
- }
-
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
-
- // Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
- Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
-
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
- }
-
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
- Collection<ServerName> servers = clusterStatus.getServers();
- final long megaByte = 1024L * 1024L;
-
- // Iterate all cluster regions, filter regions from our table and
- // compute their size.
- for (ServerName serverName : servers) {
- ServerLoad serverLoad = clusterStatus.getLoad(serverName);
-
- for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
- byte[] regionId = regionLoad.getName();
-
- if (tableRegions.contains(regionId)) {
-
- long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
- sizeMap.put(regionId, regionSizeBytes);
-
- // logger.info("Region " + regionLoad.getNameAsString()
- // + " has size " + regionSizeBytes);
- }
- }
- }
- } finally {
- hBaseAdmin.close();
- }
-
- }
-
- boolean enabled(Configuration configuration) {
- return configuration.getBoolean(ENABLE_REGIONSIZECALCULATOR, true);
- }
-
- /**
- * Returns size of given region in bytes. Returns 0 if region was not found.
- **/
- public long getRegionSize(byte[] regionId) {
- Long size = sizeMap.get(regionId);
- if (size == null) {
- logger.info("Unknown region:" + Arrays.toString(regionId));
- return 0;
- } else {
- return size;
- }
- }
-
- public Map<byte[], Long> getRegionSizeMap() {
- return Collections.unmodifiableMap(sizeMap);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
deleted file mode 100644
index 483a7a1..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.kylin.common.KylinConfig;
-
-public class HadoopUtil {
-
- private static ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
-
- private static ThreadLocal<Configuration> hbaseConfig = new ThreadLocal<>();
-
- public static void setCurrentConfiguration(Configuration conf) {
- hadoopConfig.set(conf);
- }
-
- public static void setCurrentHBaseConfiguration(Configuration conf) {
- hbaseConfig.set(conf);
- }
-
- public static Configuration getCurrentConfiguration() {
- if (hadoopConfig.get() == null) {
- hadoopConfig.set(newConfiguration());
- }
- return hadoopConfig.get();
- }
-
- public static Configuration getCurrentHBaseConfiguration() {
- if (hbaseConfig.get() == null) {
- String storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl();
- hbaseConfig.set(newHBaseConfiguration(storageUrl));
- }
- return hbaseConfig.get();
- }
-
- public static FileSystem getFileSystem(String path) throws IOException {
- return FileSystem.get(makeURI(path), getCurrentConfiguration());
- }
-
- public static URI makeURI(String filePath) {
- try {
- return new URI(fixWindowsPath(filePath));
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
- }
- }
-
- public static String fixWindowsPath(String path) {
- // fix windows path
- if (path.startsWith("file://") && !path.startsWith("file:///") && path.contains(":\\")) {
- path = path.replace("file://", "file:///");
- }
- if (path.startsWith("file:///")) {
- path = path.replace('\\', '/');
- }
- return path;
- }
-
- public static String makeQualifiedPathInHadoopCluster(String path) {
- try {
- FileSystem fs = FileSystem.get(getCurrentConfiguration());
- return fs.makeQualified(new Path(path)).toString();
- } catch (IOException e) {
- throw new IllegalArgumentException("Cannot create FileSystem from current hadoop cluster conf", e);
- }
- }
-
- public static String makeQualifiedPathInHBaseCluster(String path) {
- try {
- FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
- return fs.makeQualified(new Path(path)).toString();
- } catch (IOException e) {
- throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e);
- }
- }
-
- public static Configuration newConfiguration() {
- Configuration conf = new Configuration();
- return healSickConfig(conf);
- }
-
- public static Configuration newHBaseConfiguration(String url) {
- Configuration conf = HBaseConfiguration.create(getCurrentConfiguration());
-
- // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
- if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
- throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
- // support hbase using a different FS
- String hbaseClusterFs = KylinConfig.getInstanceFromEnv().getHBaseClusterFs();
- if (StringUtils.isNotEmpty(hbaseClusterFs)) {
- conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hbaseClusterFs);
- }
-
- // reduce rpc retry
- conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
- conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
- // conf.set(ScannerCallable.LOG_SCANNER_ACTIVITY, "true");
-
- return healSickConfig(conf);
- }
-
- private static Configuration healSickConfig(Configuration conf) {
- // https://issues.apache.org/jira/browse/KYLIN-953
- if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
- conf.set("hadoop.tmp.dir", "/tmp");
- }
- if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) {
- conf.set("hbase.fs.tmp.dir", "/tmp");
- }
- return conf;
- }
-
- /**
- *
- * @param table the identifier of hive table, in format <db_name>.<table_name>
- * @return a string array with 2 elements: {"db_name", "table_name"}
- */
- public static String[] parseHiveTableName(String table) {
- int cut = table.indexOf('.');
- String database = cut >= 0 ? table.substring(0, cut).trim() : "DEFAULT";
- String tableName = cut >= 0 ? table.substring(cut + 1).trim() : table.trim();
-
- return new String[] { database, tableName };
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
deleted file mode 100644
index c56cb6c..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-/**
- * Hive meta API client for Kylin
- * @author shaoshi
- *
- */
-public class HiveClient {
-
- protected HiveConf hiveConf = null;
- protected Driver driver = null;
- protected HiveMetaStoreClient metaStoreClient = null;
- protected String type;
-
- public HiveClient() {
- hiveConf = new HiveConf(HiveClient.class);
- }
-
- public HiveClient(Map<String, String> configMap) {
- this();
- appendConfiguration(configMap);
- }
-
- public HiveConf getHiveConf() {
- return hiveConf;
- }
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
-
- /**
- * Append or overwrite the default hive client configuration; You need call this before invoke #executeHQL;
- * @param configMap
- */
- public void appendConfiguration(Map<String, String> configMap) {
- if (configMap != null && configMap.size() > 0) {
- for (Entry<String, String> e : configMap.entrySet()) {
- hiveConf.set(e.getKey(), e.getValue());
- }
- }
- }
-
- /**
- *
- * @param hql
- * @throws org.apache.hadoop.hive.ql.CommandNeedRetryException
- * @throws java.io.IOException
- */
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
- }
- }
-
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
- for (String sql : hqls)
- executeHQL(sql);
- }
-
- private HiveMetaStoreClient getMetaStoreClient() throws Exception {
- if (metaStoreClient == null) {
- metaStoreClient = new HiveMetaStoreClient(hiveConf);
- }
- return metaStoreClient;
- }
-
- public Table getHiveTable(String database, String tableName) throws Exception {
- return getMetaStoreClient().getTable(database, tableName);
- }
-
- public List<FieldSchema> getHiveTableFields(String database, String tableName) throws Exception {
- return getMetaStoreClient().getFields(database, tableName);
- }
-
- public String getHiveTableLocation(String database, String tableName) throws Exception {
- Table t = getHiveTable(database, tableName);
- return t.getSd().getLocation();
- }
-
- public long getFileSizeForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE);
- }
-
- public long getFileNumberForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
- }
-
- public List<String> getHiveDbNames() throws Exception {
- return getMetaStoreClient().getAllDatabases();
- }
-
- public List<String> getHiveTableNames(String database) throws Exception {
- return getMetaStoreClient().getAllTables(database);
- }
-
- /**
- * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
- *
- * Get basic stats of table
- * @param table
- * - table
- * @param statType
- * - type of stats
- * @return value of stats
- */
- public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
- Map<String, String> params = table.getParameters();
- long result = 0;
-
- if (params != null) {
- try {
- result = Long.parseLong(params.get(statType));
- } catch (NumberFormatException e) {
- result = 0;
- }
- }
- return result;
- }
-
- public boolean isNativeTable(String database, String tableName) throws Exception {
- return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
deleted file mode 100644
index 9b76899..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-public class JsonUtil {
-
- // reuse the object mapper to save memory footprint
- private static final ObjectMapper mapper = new ObjectMapper();
- private static final ObjectMapper indentMapper = new ObjectMapper();
-
- static {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true);
- }
-
- public static <T> T readValue(File src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(String content, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(content, valueType);
- }
-
- public static <T> T readValue(Reader src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(InputStream src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static <T> T readValue(byte[] src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
- return mapper.readValue(src, valueType);
- }
-
- public static void writeValueIndent(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
- indentMapper.writeValue(out, value);
- }
-
- public static void writeValue(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
- mapper.writeValue(out, value);
- }
-
- public static String writeValueAsString(Object value) throws JsonProcessingException {
- return mapper.writeValueAsString(value);
- }
-
- public static byte[] writeValueAsBytes(Object value) throws JsonProcessingException {
- return mapper.writeValueAsBytes(value);
- }
-
- public static String writeValueAsIndentString(Object value) throws JsonProcessingException {
- return indentMapper.writeValueAsString(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
deleted file mode 100644
index 8653a6c..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Enumeration;
-
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-
-/**
- * Created by dongli on 11/24/15.
- */
-public class Log4jConfigurer {
- private final static String DEFAULT_PATTERN_LAYOUT = "L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n";
- private static boolean INITIALIZED = false;
-
- public static void initLogger() {
- if (!INITIALIZED && !isConfigured()) {
- org.apache.log4j.BasicConfigurator.configure(new ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT)));
- }
- INITIALIZED = true;
- }
-
- private static boolean isConfigured() {
- if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) {
- return true;
- } else {
- Enumeration<?> loggers = LogManager.getCurrentLoggers();
- while (loggers.hasMoreElements()) {
- Logger logger = (Logger) loggers.nextElement();
- if (logger.getAllAppenders().hasMoreElements())
- return true;
- }
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java b/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
deleted file mode 100644
index e8a8dfb..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/LogTitlePrinter.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/27/15.
- */
-public class LogTitlePrinter {
- public static void printTitle(String title) {
- String leftAlignFormat = "| %-100s | %n";
-
- System.out.format("+------------------------------------------------------------------------------------------------------+%n");
- System.out.format(leftAlignFormat, title);
- System.out.format("+------------------------------------------------------------------------------------------------------+%n");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Logger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Logger.java b/common/src/main/java/org/apache/kylin/common/util/Logger.java
deleted file mode 100644
index 0982b7b..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Logger.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author ysong1
- *
- */
-public interface Logger {
- public void log(String message);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/MailService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MailService.java b/common/src/main/java/org/apache/kylin/common/util/MailService.java
deleted file mode 100644
index f3e385b..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.mail.Email;
-import org.apache.commons.mail.EmailException;
-import org.apache.commons.mail.HtmlEmail;
-import org.apache.kylin.common.KylinConfig;
-
-/**
- * @author xduo
- */
-public class MailService {
-
- private Boolean enabled = Boolean.TRUE;
- private String host;
- private String username;
- private String password;
- private String sender;
-
- private static final Log logger = LogFactory.getLog(MailService.class);
-
- public MailService() {
- this(KylinConfig.getInstanceFromEnv());
- }
-
- public MailService(KylinConfig config) {
- enabled = "true".equalsIgnoreCase(config.getProperty(KylinConfig.MAIL_ENABLED, "false"));
- host = config.getProperty(KylinConfig.MAIL_HOST, "");
- username = config.getProperty(KylinConfig.MAIL_USERNAME, "");
- password = config.getProperty(KylinConfig.MAIL_PASSWORD, "");
- sender = config.getProperty(KylinConfig.MAIL_SENDER, "");
-
- if (enabled) {
- if (host.isEmpty())
- throw new RuntimeException("mail service host is empty");
- }
- }
-
- /**
- * @param receivers
- * @param subject
- * @param content
- * @return true or false indicating whether the email was delivered successfully
- * @throws IOException
- */
- public boolean sendMail(List<String> receivers, String subject, String content) throws IOException {
-
- if (!enabled) {
- logger.info("Email service is disabled; this mail will not be delivered: " + subject);
- logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties");
- return false;
- }
-
- Email email = new HtmlEmail();
- email.setHostName(host);
- if (username != null && username.trim().length() > 0) {
- email.setAuthentication(username, password);
- }
-
- //email.setDebug(true);
- try {
- for (String receiver : receivers) {
- email.addTo(receiver);
- }
-
- email.setFrom(sender);
- email.setSubject(subject);
- email.setCharset("UTF-8");
- ((HtmlEmail) email).setHtmlMsg(content);
- email.send();
- email.getMailSession();
-
- } catch (EmailException e) {
- logger.error(e.getLocalizedMessage(), e);
- return false;
- }
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java b/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java
deleted file mode 100644
index b4af6ce..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/MyLogFormatter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.sql.Timestamp;
-import java.util.Date;
-import java.util.logging.Formatter;
-import java.util.logging.LogRecord;
-
-public class MyLogFormatter extends Formatter {
-
- Date dat = new Date();
-
- // Line separator string. This is the value of the line.separator
- // property at the moment that the SimpleFormatter was created.
- private String lineSeparator = "\n";
-
- /**
- * Format the given LogRecord.
- *
- * @param record
- * the log record to be formatted.
- * @return a formatted log record
- */
- public synchronized String format(LogRecord record) {
- StringBuffer sb = new StringBuffer();
- // Minimize memory allocations here.
- Timestamp ts = new Timestamp(record.getMillis());
- String text = ts.toString();
- sb.append("JUL ");
- sb.append(text);
- sb.append(" ");
- if (record.getSourceClassName() != null) {
- sb.append(record.getSourceClassName());
- } else {
- sb.append(record.getLoggerName());
- }
- if (record.getSourceMethodName() != null) {
- sb.append(" ");
- sb.append(record.getSourceMethodName());
- }
- sb.append(lineSeparator);
- String message = formatMessage(record);
- sb.append(record.getLevel().getLocalizedName());
- sb.append(": ");
- sb.append(message);
- sb.append(lineSeparator);
- if (record.getThrown() != null) {
- try {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- record.getThrown().printStackTrace(pw);
- pw.close();
- sb.append(sw.toString());
- } catch (Exception ex) {
- }
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/Pair.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/Pair.java b/common/src/main/java/org/apache/kylin/common/util/Pair.java
deleted file mode 100644
index 17056af..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/Pair.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-
-/**
- * A generic class for pairs. Copied from org.apache.hadoop.hbase.util.Pair
- * @param <T1>
- * @param <T2>
- */
-public class Pair<T1, T2> implements Serializable {
- private static final long serialVersionUID = -3986244606585552569L;
- protected T1 first = null;
- protected T2 second = null;
-
- /**
- * Default constructor.
- */
- public Pair() {
- }
-
- /**
- * Constructor
- * @param a operand
- * @param b operand
- */
- public Pair(T1 a, T2 b) {
- this.first = a;
- this.second = b;
- }
-
- /**
- * Constructs a new pair, inferring the type via the passed arguments
- * @param <T1> type for first
- * @param <T2> type for second
- * @param a first element
- * @param b second element
- * @return a new pair containing the passed arguments
- */
- public static <T1, T2> Pair<T1, T2> newPair(T1 a, T2 b) {
- return new Pair<T1, T2>(a, b);
- }
-
- /**
- * Replace the first element of the pair.
- * @param a operand
- */
- public void setFirst(T1 a) {
- this.first = a;
- }
-
- /**
- * Replace the second element of the pair.
- * @param b operand
- */
- public void setSecond(T2 b) {
- this.second = b;
- }
-
- /**
- * Return the first element stored in the pair.
- * @return T1
- */
- public T1 getFirst() {
- return first;
- }
-
- /**
- * Return the second element stored in the pair.
- * @return T2
- */
- public T2 getSecond() {
- return second;
- }
-
- private static boolean equals(Object x, Object y) {
- return (x == null && y == null) || (x != null && x.equals(y));
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public boolean equals(Object other) {
- return other instanceof Pair && equals(first, ((Pair) other).first) && equals(second, ((Pair) other).second);
- }
-
- @Override
- public int hashCode() {
- if (first == null)
- return (second == null) ? 0 : second.hashCode() + 1;
- else if (second == null)
- return first.hashCode() + 2;
- else
- return first.hashCode() * 17 + second.hashCode();
- }
-
- @Override
- public String toString() {
- return "{" + getFirst() + "," + getSecond() + "}";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java b/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
deleted file mode 100644
index a580302..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
- *
- * This utility class sorts only the specified part of a list
- */
-public class PartialSorter {
- public static <T> void partialSort(List<T> list, List<Integer> items, Comparator<? super T> c) {
- List<T> temp = Lists.newLinkedList();
- for (int index : items) {
- temp.add(list.get(index));
- }
- Collections.sort(temp, c);
- for (int i = 0; i < temp.size(); ++i) {
- list.set(items.get(i), temp.get(i));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java b/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
deleted file mode 100644
index e0a31e4..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * @author ysong1
- *
- */
-public class RandomSampler<T> {
-
- private Random rdm = new Random();
-
- public List<T> sample(List<T> data, int sampleNumber) {
- if (data == null) {
- throw new IllegalArgumentException("Input list is null");
- }
- if (data.size() < sampleNumber) {
- return data;
- }
-
- List<T> result = new ArrayList<T>(sampleNumber);
- int n = data.size();
- for (int i = 0; i < n; i++) {
- if (i < sampleNumber) {
- result.add(data.get(i));
- } else {
- int j = rdm.nextInt(i);
- if (j < sampleNumber) {
- result.set(j, data.get(i));
- }
- }
- }
- return result;
- }
-}
[33/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
deleted file mode 100644
index 9727de7..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ /dev/null
@@ -1,536 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * Builds a dictionary using Trie structure. All values are taken in byte[] form
- * and organized in a Trie with ordering. Then numeric IDs are assigned in
- * sequence.
- *
- * @author yangli9
- */
-public class TrieDictionaryBuilder<T> {
-
- public static class Node {
- public byte[] part;
- public boolean isEndOfValue;
- public ArrayList<Node> children;
-
- public int nValuesBeneath; // only present after stats()
-
- Node(byte[] value, boolean isEndOfValue) {
- reset(value, isEndOfValue);
- }
-
- Node(byte[] value, boolean isEndOfValue, ArrayList<Node> children) {
- reset(value, isEndOfValue, children);
- }
-
- void reset(byte[] value, boolean isEndOfValue) {
- reset(value, isEndOfValue, new ArrayList<Node>());
- }
-
- void reset(byte[] value, boolean isEndOfValue, ArrayList<Node> children) {
- this.part = value;
- this.isEndOfValue = isEndOfValue;
- this.children = children;
- }
- }
-
- public static interface Visitor {
- void visit(Node n, int level);
- }
-
- // ============================================================================
-
- private Node root;
- private BytesConverter<T> bytesConverter;
-
- public TrieDictionaryBuilder(BytesConverter<T> bytesConverter) {
- this.root = new Node(new byte[0], false);
- this.bytesConverter = bytesConverter;
- }
-
- public void addValue(T value) {
- addValue(bytesConverter.convertToBytes(value));
- }
-
- public void addValue(byte[] value) {
- addValueR(root, value, 0);
- }
-
- private void addValueR(Node node, byte[] value, int start) {
- // match the value part of current node
- int i = 0, j = start;
- int n = node.part.length, nn = value.length;
- int comp = 0;
- for (; i < n && j < nn; i++, j++) {
- comp = BytesUtil.compareByteUnsigned(node.part[i], value[j]);
- if (comp != 0)
- break;
- }
-
- // if value fully matched within the current node
- if (j == nn) {
- // if equals to current node, just mark end of value
- if (i == n) {
- node.isEndOfValue = true;
- }
- // otherwise, split the current node into two
- else {
- Node c = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
- node.reset(BytesUtil.subarray(node.part, 0, i), true);
- node.children.add(c);
- }
- return;
- }
-
- // if partially matched the current, split the current node, add the new
- // value, make a 3-way
- if (i < n) {
- Node c1 = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
- Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
- node.reset(BytesUtil.subarray(node.part, 0, i), false);
- if (comp < 0) {
- node.children.add(c1);
- node.children.add(c2);
- } else {
- node.children.add(c2);
- node.children.add(c1);
- }
- return;
- }
-
- // out matched the current, binary search the next byte for a child node
- // to continue
- byte lookfor = value[j];
- int lo = 0;
- int hi = node.children.size() - 1;
- int mid = 0;
- boolean found = false;
- comp = 0;
- while (!found && lo <= hi) {
- mid = lo + (hi - lo) / 2;
- comp = BytesUtil.compareByteUnsigned(lookfor, node.children.get(mid).part[0]);
- if (comp < 0)
- hi = mid - 1;
- else if (comp > 0)
- lo = mid + 1;
- else
- found = true;
- }
- // found a child node matching the first byte, continue in that child
- if (found) {
- addValueR(node.children.get(mid), value, j);
- }
- // otherwise, make the value a new child
- else {
- Node c = new Node(BytesUtil.subarray(value, j, nn), true);
- node.children.add(comp <= 0 ? mid : mid + 1, c);
- }
- }
-
- public void traverse(Visitor visitor) {
- traverseR(root, visitor, 0);
- }
-
- private void traverseR(Node node, Visitor visitor, int level) {
- visitor.visit(node, level);
- for (Node c : node.children)
- traverseR(c, visitor, level + 1);
- }
-
- public void traversePostOrder(Visitor visitor) {
- traversePostOrderR(root, visitor, 0);
- }
-
- private void traversePostOrderR(Node node, Visitor visitor, int level) {
- for (Node c : node.children)
- traversePostOrderR(c, visitor, level + 1);
- visitor.visit(node, level);
- }
-
- public static class Stats {
- public int nValues; // number of values in total
- public int nValueBytesPlain; // number of bytes for all values
- // uncompressed
- public int nValueBytesCompressed; // number of values bytes in Trie
- // (compressed)
- public int maxValueLength; // size of longest value in bytes
-
- // the trie is multi-byte-per-node
- public int mbpn_nNodes; // number of nodes in trie
- public int mbpn_trieDepth; // depth of trie
- public int mbpn_maxFanOut; // the maximum no. children
- public int mbpn_nChildLookups; // number of child lookups during lookup
- // every value once
- public int mbpn_nTotalFanOut; // the sum of fan outs during lookup every
- // value once
- public int mbpn_sizeValueTotal; // the sum of value space in all nodes
- public int mbpn_sizeNoValueBytes; // size of field noValueBytes
- public int mbpn_sizeNoValueBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int mbpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
- public int mbpn_footprint; // MBPN footprint in bytes
-
- // stats for one-byte-per-node as well, so there's comparison
- public int obpn_sizeValue; // size of value per node, always 1
- public int obpn_sizeNoValuesBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int obpn_sizeChildCount; // size of field childCount, enables
- // binary search among children
- public int obpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
- public int obpn_nNodes; // no. nodes in OBPN trie
- public int obpn_footprint; // OBPN footprint in bytes
-
- public void print() {
- PrintStream out = System.out;
- out.println("============================================================================");
- out.println("No. values: " + nValues);
- out.println("No. bytes raw: " + nValueBytesPlain);
- out.println("No. bytes in trie: " + nValueBytesCompressed);
- out.println("Longest value length: " + maxValueLength);
-
- // flatten trie footprint calculation, case of One-Byte-Per-Node
- out.println("----------------------------------------------------------------------------");
- out.println("OBPN node size: " + (obpn_sizeValue + obpn_sizeNoValuesBeneath + obpn_sizeChildCount + obpn_sizeChildOffset) + " = " + obpn_sizeValue + " + " + obpn_sizeNoValuesBeneath + " + " + obpn_sizeChildCount + " + " + obpn_sizeChildOffset);
- out.println("OBPN no. nodes: " + obpn_nNodes);
- out.println("OBPN trie depth: " + maxValueLength);
- out.println("OBPN footprint: " + obpn_footprint + " in bytes");
-
- // flatten trie footprint calculation, case of Multi-Byte-Per-Node
- out.println("----------------------------------------------------------------------------");
- out.println("MBPN max fan out: " + mbpn_maxFanOut);
- out.println("MBPN no. child lookups: " + mbpn_nChildLookups);
- out.println("MBPN total fan out: " + mbpn_nTotalFanOut);
- out.println("MBPN average fan out: " + (double) mbpn_nTotalFanOut / mbpn_nChildLookups);
- out.println("MBPN values size total: " + mbpn_sizeValueTotal);
- out.println("MBPN node size: " + (mbpn_sizeNoValueBytes + mbpn_sizeNoValueBeneath + mbpn_sizeChildOffset) + " = " + mbpn_sizeNoValueBytes + " + " + mbpn_sizeNoValueBeneath + " + " + mbpn_sizeChildOffset);
- out.println("MBPN no. nodes: " + mbpn_nNodes);
- out.println("MBPN trie depth: " + mbpn_trieDepth);
- out.println("MBPN footprint: " + mbpn_footprint + " in bytes");
- }
- }
-
- /** out print some statistics of the trie and the dictionary built from it */
- public Stats stats() {
- // calculate nEndValueBeneath
- traversePostOrder(new Visitor() {
- @Override
- public void visit(Node n, int level) {
- n.nValuesBeneath = n.isEndOfValue ? 1 : 0;
- for (Node c : n.children)
- n.nValuesBeneath += c.nValuesBeneath;
- }
- });
-
- // run stats
- final Stats s = new Stats();
- final ArrayList<Integer> lenAtLvl = new ArrayList<Integer>();
- traverse(new Visitor() {
- @Override
- public void visit(Node n, int level) {
- if (n.isEndOfValue)
- s.nValues++;
- s.nValueBytesPlain += n.part.length * n.nValuesBeneath;
- s.nValueBytesCompressed += n.part.length;
- s.mbpn_nNodes++;
- if (s.mbpn_trieDepth < level + 1)
- s.mbpn_trieDepth = level + 1;
- if (n.children.size() > 0) {
- if (s.mbpn_maxFanOut < n.children.size())
- s.mbpn_maxFanOut = n.children.size();
- int childLookups = n.nValuesBeneath - (n.isEndOfValue ? 1 : 0);
- s.mbpn_nChildLookups += childLookups;
- s.mbpn_nTotalFanOut += childLookups * n.children.size();
- }
-
- if (level < lenAtLvl.size())
- lenAtLvl.set(level, n.part.length);
- else
- lenAtLvl.add(n.part.length);
- int lenSoFar = 0;
- for (int i = 0; i <= level; i++)
- lenSoFar += lenAtLvl.get(i);
- if (lenSoFar > s.maxValueLength)
- s.maxValueLength = lenSoFar;
- }
- });
-
- // flatten trie footprint calculation, case of One-Byte-Per-Node
- s.obpn_sizeValue = 1;
- s.obpn_sizeNoValuesBeneath = BytesUtil.sizeForValue(s.nValues);
- s.obpn_sizeChildCount = 1;
- s.obpn_sizeChildOffset = 4; // MSB used as isEndOfValue flag
- s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total
- // number of compressed
- // bytes in OBPN
- s.obpn_footprint = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
- while (true) { // minimize the offset size to match the footprint
- int t = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2
- // because
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // flag
- s.obpn_sizeChildOffset--;
- s.obpn_footprint = t;
- } else
- break;
- }
-
- // flatten trie footprint calculation, case of Multi-Byte-Per-Node
- s.mbpn_sizeValueTotal = s.nValueBytesCompressed;
- s.mbpn_sizeNoValueBytes = 1;
- s.mbpn_sizeNoValueBeneath = BytesUtil.sizeForValue(s.nValues);
- s.mbpn_sizeChildOffset = 4;
- s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
- while (true) { // minimize the offset size to match the footprint
- int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4
- // because
- // 2
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // &
- // isEndChild
- // flag
- s.mbpn_sizeChildOffset--;
- s.mbpn_footprint = t;
- } else
- break;
- }
-
- return s;
- }
-
- /** out print trie for debug */
- public void print() {
- print(System.out);
- }
-
- public void print(final PrintStream out) {
- traverse(new Visitor() {
- @Override
- public void visit(Node n, int level) {
- try {
- for (int i = 0; i < level; i++)
- out.print(" ");
- out.print(new String(n.part, "UTF-8"));
- out.print(" - ");
- if (n.nValuesBeneath > 0)
- out.print(n.nValuesBeneath);
- if (n.isEndOfValue)
- out.print("*");
- out.print("\n");
- } catch (UnsupportedEncodingException e) {
- e.printStackTrace();
- }
- }
- });
- }
-
- private CompleteParts completeParts = new CompleteParts();
-
- private class CompleteParts {
- byte[] data = new byte[4096];
- int current = 0;
-
- public void append(byte[] part) {
- while (current + part.length > data.length)
- expand();
-
- System.arraycopy(part, 0, data, current, part.length);
- current += part.length;
- }
-
- public void withdraw(int size) {
- current -= size;
- }
-
- public byte[] retrieve() {
- return Arrays.copyOf(data, current);
- }
-
- private void expand() {
- byte[] temp = new byte[2 * data.length];
- System.arraycopy(data, 0, temp, 0, data.length);
- data = temp;
- }
- }
-
- // there is a 255 limitation of length for each node's part.
- // we interpolate nodes to satisfy this when a node's part becomes
- // too long(overflow)
- private void checkOverflowParts(Node node) {
- LinkedList<Node> childrenCopy = new LinkedList<Node>(node.children);
- for (Node child : childrenCopy) {
- if (child.part.length > 255) {
- byte[] first255 = Arrays.copyOf(child.part, 255);
-
- completeParts.append(node.part);
- completeParts.append(first255);
- byte[] visited = completeParts.retrieve();
- this.addValue(visited);
- completeParts.withdraw(255);
- completeParts.withdraw(node.part.length);
- }
- }
-
- completeParts.append(node.part);// by here the node.children may have
- // been changed
- for (Node child : node.children) {
- checkOverflowParts(child);
- }
- completeParts.withdraw(node.part.length);
- }
-
- /**
- * Flatten the trie into a byte array for a minimized memory footprint.
- * Lookup remains fast. Cost is inflexibility to modify (becomes immutable).
- *
- * Flattened node structure is HEAD + NODEs, for each node: - o byte, offset
- * to child node, o = stats.mbpn_sizeChildOffset - 1 bit, isLastChild flag,
- * the 1st MSB of o - 1 bit, isEndOfValue flag, the 2nd MSB of o - c byte,
- * number of values beneath, c = stats.mbpn_sizeNoValueBeneath - 1 byte,
- * number of value bytes - n byte, value bytes
- */
- public TrieDictionary<T> build(int baseId) {
- byte[] trieBytes = buildTrieBytes(baseId);
- TrieDictionary<T> r = new TrieDictionary<T>(trieBytes);
- return r;
- }
-
- protected byte[] buildTrieBytes(int baseId) {
- checkOverflowParts(this.root);
-
- Stats stats = stats();
- int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
- int sizeChildOffset = stats.mbpn_sizeChildOffset;
-
- // write head
- byte[] head;
- try {
- ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
- DataOutputStream headOut = new DataOutputStream(byteBuf);
- headOut.write(TrieDictionary.HEAD_MAGIC);
- headOut.writeShort(0); // head size, will back fill
- headOut.writeInt(stats.mbpn_footprint); // body size
- headOut.write(sizeChildOffset);
- headOut.write(sizeNoValuesBeneath);
- headOut.writeShort(baseId);
- headOut.writeShort(stats.maxValueLength);
- headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
- headOut.close();
- head = byteBuf.toByteArray();
- BytesUtil.writeUnsigned(head.length, head, TrieDictionary.HEAD_SIZE_I, 2);
- } catch (IOException e) {
- throw new RuntimeException(e); // shall not happen, as we are
- // writing in memory
- }
-
- byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
- System.arraycopy(head, 0, trieBytes, 0, head.length);
-
- LinkedList<Node> open = new LinkedList<Node>();
- IdentityHashMap<Node, Integer> offsetMap = new IdentityHashMap<Node, Integer>();
-
- // write body
- int o = head.length;
- offsetMap.put(root, o);
- o = build_writeNode(root, o, true, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
- if (!root.children.isEmpty())
- open.addLast(root);
-
- while (!open.isEmpty()) {
- Node parent = open.removeFirst();
- build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
- for (int i = 0; i < parent.children.size(); i++) {
- Node c = parent.children.get(i);
- boolean isLastChild = (i == parent.children.size() - 1);
- offsetMap.put(c, o);
- o = build_writeNode(c, o, isLastChild, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
- if (!c.children.isEmpty())
- open.addLast(c);
- }
- }
-
- if (o != trieBytes.length)
- throw new RuntimeException();
- return trieBytes;
- }
-
- private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
- int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
- BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
- trieBytes[parentOffset] |= flags;
- }
-
- private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
- int o = offset;
-
- // childOffset
- if (isLastChild)
- trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
- if (n.isEndOfValue)
- trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
- o += sizeChildOffset;
-
- // nValuesBeneath
- BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
- o += sizeNoValuesBeneath;
-
- // nValueBytes
- if (n.part.length > 255)
- throw new RuntimeException();
- BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
- o++;
-
- // valueBytes
- System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
- o += n.part.length;
-
- return o;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
deleted file mode 100644
index 3769302..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.Pair;
-
-/**
- * @author yangli9
- *
- */
-public class FileTable implements ReadableTable {
-
- public static final String DELIM_AUTO = "auto";
- public static final String DELIM_COMMA = ",";
-
- String path;
- String delim;
- int nColumns;
-
- public FileTable(String path, int nColumns) {
- this(path, DELIM_AUTO, nColumns);
- }
-
- public FileTable(String path, String delim, int nColumns) {
- this.path = path;
- this.delim = delim;
- this.nColumns = nColumns;
- }
-
- @Override
- public TableReader getReader() throws IOException {
- return new FileTableReader(path, delim, nColumns);
- }
-
- @Override
- public TableSignature getSignature() throws IOException {
- if (!exists())
- throw new IllegalStateException("Table not exists :" + path);
-
- try {
- Pair<Long, Long> sizeAndLastModified = getSizeAndLastModified(path);
- return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
- } catch (FileNotFoundException ex) {
- return null;
- }
- }
-
- @Override
- public boolean exists() throws IOException {
- FileSystem fs = HadoopUtil.getFileSystem(path);
- return fs.exists(new Path(path));
- }
-
- @Override
- public String toString() {
- return path;
- }
-
- public static Pair<Long, Long> getSizeAndLastModified(String path) throws IOException {
- FileSystem fs = HadoopUtil.getFileSystem(path);
-
- // get all contained files if path is directory
- ArrayList<FileStatus> allFiles = new ArrayList<>();
- FileStatus status = fs.getFileStatus(new Path(path));
- if (status.isFile()) {
- allFiles.add(status);
- } else {
- FileStatus[] listStatus = fs.listStatus(new Path(path));
- allFiles.addAll(Arrays.asList(listStatus));
- }
-
- long size = 0;
- long lastModified = 0;
- for (FileStatus file : allFiles) {
- size += file.getLen();
- lastModified = Math.max(lastModified, file.getModificationTime());
- }
-
- return new Pair<Long, Long>(size, lastModified);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
deleted file mode 100644
index a6de5df..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tables are typically CSV or SEQ file.
- *
- * @author yangli9
- */
-public class FileTableReader implements TableReader {
-
- private static final Logger logger = LoggerFactory.getLogger(FileTableReader.class);
- private static final char CSV_QUOTE = '"';
- private static final String[] DETECT_DELIMS = new String[] { "\177", "|", "\t", "," };
-
- private String filePath;
- private String delim;
- private RowReader reader;
-
- private String curLine;
- private String[] curColumns;
- private int expectedColumnNumber = -1; // helps delimiter detection
-
- public FileTableReader(String filePath, String delim, int expectedColumnNumber) throws IOException {
- this.filePath = filePath;
- this.delim = delim;
- this.expectedColumnNumber = expectedColumnNumber;
-
- FileSystem fs = HadoopUtil.getFileSystem(filePath);
-
- try {
- this.reader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, filePath);
-
- } catch (IOException e) {
- if (isExceptionSayingNotSeqFile(e) == false)
- throw e;
-
- this.reader = new CsvRowReader(fs, filePath);
- }
- }
-
- private boolean isExceptionSayingNotSeqFile(IOException e) {
- if (e.getMessage() != null && e.getMessage().contains("not a SequenceFile"))
- return true;
-
- if (e instanceof EOFException) // in case the file is very very small
- return true;
-
- return false;
- }
-
- @Override
- public boolean next() throws IOException {
- curLine = reader.nextLine();
- curColumns = null;
- return curLine != null;
- }
-
- public String getLine() {
- return curLine;
- }
-
- @Override
- public String[] getRow() {
- if (curColumns == null) {
- if (FileTable.DELIM_AUTO.equals(delim))
- delim = autoDetectDelim(curLine);
-
- if (delim == null)
- curColumns = new String[] { curLine };
- else
- curColumns = split(curLine, delim);
- }
- return curColumns;
- }
-
- private String[] split(String line, String delim) {
- // FIXME CVS line should be parsed considering escapes
- String str[] = StringSplitter.split(line, delim);
-
- // un-escape CSV
- if (FileTable.DELIM_COMMA.equals(delim)) {
- for (int i = 0; i < str.length; i++) {
- str[i] = unescapeCsv(str[i]);
- }
- }
-
- return str;
- }
-
- private String unescapeCsv(String str) {
- if (str == null || str.length() < 2)
- return str;
-
- str = StringEscapeUtils.unescapeCsv(str);
-
- // unescapeCsv may not remove the outer most quotes
- if (str.charAt(0) == CSV_QUOTE && str.charAt(str.length() - 1) == CSV_QUOTE)
- str = str.substring(1, str.length() - 1);
-
- return str;
- }
-
- @Override
- public void close() throws IOException {
- if (reader != null)
- reader.close();
- }
-
- private String autoDetectDelim(String line) {
- if (expectedColumnNumber > 0) {
- for (String delim : DETECT_DELIMS) {
- if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
- logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
- return delim;
- }
- }
- }
-
- logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
- return null;
- }
-
- // ============================================================================
-
- private interface RowReader extends Closeable {
- String nextLine() throws IOException; // return null on EOF
- }
-
- private class SeqRowReader implements RowReader {
- Reader reader;
- Writable key;
- Text value;
-
- SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException {
- path = HadoopUtil.fixWindowsPath(path);
- reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path)));
- key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- value = new Text();
- }
-
- @Override
- public String nextLine() throws IOException {
- boolean hasNext = reader.next(key, value);
- if (hasNext)
- return Bytes.toString(value.getBytes(), 0, value.getLength());
- else
- return null;
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
- }
-
- private class CsvRowReader implements RowReader {
- BufferedReader reader;
-
- CsvRowReader(FileSystem fs, String path) throws IOException {
- path = HadoopUtil.fixWindowsPath(path);
- FSDataInputStream in = fs.open(new Path(path));
- reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
- }
-
- @Override
- public String nextLine() throws IOException {
- return reader.readLine();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
deleted file mode 100644
index 0237ff4..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HiveClient;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class HiveTable implements ReadableTable {
-
- private static final Logger logger = LoggerFactory.getLogger(HiveTable.class);
-
- final private String database;
- final private String hiveTable;
-
- private HiveClient hiveClient;
-
- public HiveTable(MetadataManager metaMgr, String table) {
- TableDesc tableDesc = metaMgr.getTableDesc(table);
- this.database = tableDesc.getDatabase();
- this.hiveTable = tableDesc.getName();
- }
-
- @Override
- public TableReader getReader() throws IOException {
- return new HiveTableReader(database, hiveTable);
- }
-
- @Override
- public TableSignature getSignature() throws IOException {
- try {
- String path = computeHDFSLocation();
- Pair<Long, Long> sizeAndLastModified = FileTable.getSizeAndLastModified(path);
- long size = sizeAndLastModified.getFirst();
- long lastModified = sizeAndLastModified.getSecond();
-
- // for non-native hive table, cannot rely on size & last modified on HDFS
- if (getHiveClient().isNativeTable(database, hiveTable) == false) {
- lastModified = System.currentTimeMillis(); // assume table is ever changing
- }
-
- return new TableSignature(path, size, lastModified);
-
- } catch (Exception e) {
- if (e instanceof IOException)
- throw (IOException) e;
- else
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean exists() throws IOException {
- return true;
- }
-
- private String computeHDFSLocation() throws Exception {
-
- String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable);
- if (override != null) {
- logger.debug("Override hive table location " + hiveTable + " -- " + override);
- return override;
- }
-
- return getHiveClient().getHiveTableLocation(database, hiveTable);
- }
-
- @Override
- public String toString() {
- return "hive: database=[" + database + "], table=[" + hiveTable + "]";
- }
-
- public HiveClient getHiveClient() {
- if (hiveClient == null) {
- hiveClient = new HiveClient();
- }
- return hiveClient;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
deleted file mode 100644
index b103442..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTableReader.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hive.hcatalog.common.HCatException;
-import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
-import org.apache.hive.hcatalog.data.transfer.HCatReader;
-import org.apache.hive.hcatalog.data.transfer.ReadEntity;
-import org.apache.hive.hcatalog.data.transfer.ReaderContext;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-
-/**
- * An implementation of TableReader with HCatalog for Hive table.
- * @author shaoshi
- *
- */
-public class HiveTableReader implements TableReader {
-
- private String dbName;
- private String tableName;
- private int currentSplit = -1;
- private ReaderContext readCntxt = null;
- private Iterator<HCatRecord> currentHCatRecordItr = null;
- private HCatRecord currentHCatRecord;
- private int numberOfSplits = 0;
- private Map<String, String> partitionKV = null;
-
- /**
- * Constructor for reading whole hive table
- * @param dbName
- * @param tableName
- * @throws IOException
- */
- public HiveTableReader(String dbName, String tableName) throws IOException {
- this(dbName, tableName, null);
- }
-
- /**
- * Constructor for reading a partition of the hive table
- * @param dbName
- * @param tableName
- * @param partitionKV key-value pairs condition on the partition
- * @throws IOException
- */
- public HiveTableReader(String dbName, String tableName, Map<String, String> partitionKV) throws IOException {
- this.dbName = dbName;
- this.tableName = tableName;
- this.partitionKV = partitionKV;
- initialize();
- }
-
- private void initialize() throws IOException {
- try {
- this.readCntxt = getHiveReaderContext(dbName, tableName, partitionKV);
- } catch (Exception e) {
- e.printStackTrace();
- throw new IOException(e);
- }
-
- this.numberOfSplits = readCntxt.numSplits();
- }
-
- @Override
- public boolean next() throws IOException {
-
- while (currentHCatRecordItr == null || !currentHCatRecordItr.hasNext()) {
- currentSplit++;
- if (currentSplit == numberOfSplits) {
- return false;
- }
-
- currentHCatRecordItr = loadHCatRecordItr(readCntxt, currentSplit);
- }
-
- currentHCatRecord = currentHCatRecordItr.next();
-
- return true;
- }
-
- @Override
- public String[] getRow() {
- List<Object> allFields = currentHCatRecord.getAll();
- List<String> rowValues = new ArrayList<String>(allFields.size());
- for (Object o : allFields) {
- rowValues.add(o != null ? o.toString() : "");
- }
-
- return rowValues.toArray(new String[allFields.size()]);
- }
-
- @Override
- public void close() throws IOException {
- this.readCntxt = null;
- this.currentHCatRecordItr = null;
- this.currentHCatRecord = null;
- this.currentSplit = -1;
- }
-
- public String toString() {
- return "hive table reader for: " + dbName + "." + tableName;
- }
-
- private static ReaderContext getHiveReaderContext(String database, String table, Map<String, String> partitionKV) throws Exception {
- HiveConf hiveConf = new HiveConf(HiveTableReader.class);
- Iterator<Entry<String, String>> itr = hiveConf.iterator();
- Map<String, String> map = new HashMap<String, String>();
- while (itr.hasNext()) {
- Entry<String, String> kv = itr.next();
- map.put(kv.getKey(), kv.getValue());
- }
-
- ReadEntity entity;
- if (partitionKV == null || partitionKV.size() == 0) {
- entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build();
- } else {
- entity = new ReadEntity.Builder().withDatabase(database).withTable(table).withPartition(partitionKV).build();
- }
-
- HCatReader reader = DataTransferFactory.getHCatReader(entity, map);
- ReaderContext cntxt = reader.prepareRead();
-
- return cntxt;
- }
-
- private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
- HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);
- return currentHCatReader.read();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
deleted file mode 100644
index d0039df..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * @author yangli9
- *
- */
-public class LookupBytesTable extends LookupTable<ByteArray> {
-
- public LookupBytesTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
- super(tableDesc, keyColumns, table);
- }
-
- @Override
- protected ByteArray[] convertRow(String[] cols) {
- ByteArray[] r = new ByteArray[cols.length];
- for (int i = 0; i < cols.length; i++) {
- r[i] = cols[i] == null ? null : new ByteArray(Bytes.toBytes(cols[i]));
- }
- return r;
- }
-
- @Override
- protected String toString(ByteArray cell) {
- return Bytes.toString(cell.data);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
deleted file mode 100644
index aaec5c7..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * @author yangli9
- *
- */
-public class LookupStringTable extends LookupTable<String> {
-
- public LookupStringTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
- super(tableDesc, keyColumns, table);
- }
-
- @Override
- protected String[] convertRow(String[] cols) {
- return cols;
- }
-
- @Override
- protected String toString(String cell) {
- return cell;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
deleted file mode 100644
index 12c8bfd..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-import org.apache.kylin.metadata.model.TableDesc;
-
-import com.google.common.collect.Sets;
-
-/**
- * An in-memory lookup table, in which each cell is an object of type T. The
- * table is indexed by specified PK for fast lookup.
- *
- * @author yangli9
- */
-abstract public class LookupTable<T extends Comparable<T>> {
-
- protected TableDesc tableDesc;
- protected String[] keyColumns;
- protected ReadableTable table;
- protected ConcurrentHashMap<Array<T>, T[]> data;
-
- public LookupTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
- this.tableDesc = tableDesc;
- this.keyColumns = keyColumns;
- this.table = table;
- this.data = new ConcurrentHashMap<Array<T>, T[]>();
- init();
- }
-
- protected void init() throws IOException {
- int[] keyIndex = new int[keyColumns.length];
- for (int i = 0; i < keyColumns.length; i++) {
- keyIndex[i] = tableDesc.findColumnByName(keyColumns[i]).getZeroBasedIndex();
- }
-
- TableReader reader = table.getReader();
- try {
- while (reader.next()) {
- initRow(reader.getRow(), keyIndex);
- }
- } finally {
- reader.close();
- }
- }
-
- @SuppressWarnings("unchecked")
- private void initRow(String[] cols, int[] keyIndex) {
- T[] value = convertRow(cols);
- T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(value[0].getClass(), keyIndex.length);
- for (int i = 0; i < keyCols.length; i++)
- keyCols[i] = value[keyIndex[i]];
-
- Array<T> key = new Array<T>(keyCols);
-
- if (data.containsKey(key))
- throw new IllegalStateException("Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));
-
- data.put(key, value);
- }
-
- abstract protected T[] convertRow(String[] cols);
-
- public T[] getRow(Array<T> key) {
- return data.get(key);
- }
-
- public Collection<T[]> getAllRows() {
- return data.values();
- }
-
- public List<T> scan(String col, List<T> values, String returnCol) {
- ArrayList<T> result = new ArrayList<T>();
- int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
- int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
- for (T[] row : data.values()) {
- if (values.contains(row[colIdx]))
- result.add(row[returnIdx]);
- }
- return result;
- }
-
- public Pair<T, T> mapRange(String col, T beginValue, T endValue, String returnCol) {
- int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
- int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
- T returnBegin = null;
- T returnEnd = null;
- for (T[] row : data.values()) {
- if (between(beginValue, row[colIdx], endValue)) {
- T returnValue = row[returnIdx];
- if (returnBegin == null || returnValue.compareTo(returnBegin) < 0) {
- returnBegin = returnValue;
- }
- if (returnEnd == null || returnValue.compareTo(returnEnd) > 0) {
- returnEnd = returnValue;
- }
- }
- }
- if (returnBegin == null && returnEnd == null)
- return null;
- else
- return new Pair<T, T>(returnBegin, returnEnd);
- }
-
- public Set<T> mapValues(String col, Set<T> values, String returnCol) {
- int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
- int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
- Set<T> result = Sets.newHashSetWithExpectedSize(values.size());
- for (T[] row : data.values()) {
- if (values.contains(row[colIdx])) {
- result.add(row[returnIdx]);
- }
- }
- return result;
- }
-
- private boolean between(T beginValue, T v, T endValue) {
- return (beginValue == null || beginValue.compareTo(v) <= 0) && (endValue == null || v.compareTo(endValue) <= 0);
- }
-
- public String toString() {
- return "LookupTable [path=" + table + "]";
- }
-
- protected String toString(T[] cols) {
- StringBuilder b = new StringBuilder();
- b.append("[");
- for (int i = 0; i < cols.length; i++) {
- if (i > 0)
- b.append(",");
- b.append(toString(cols[i]));
- }
- b.append("]");
- return b.toString();
- }
-
- abstract protected String toString(T cell);
-
- public void dump() {
- for (Array<T> key : data.keySet()) {
- System.out.println(toString(key.data) + " => " + toString(data.get(key)));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
deleted file mode 100644
index c60df21..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ReadableTable.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-public interface ReadableTable {
-
- /** Returns a reader to read the table. */
- public TableReader getReader() throws IOException;
-
- /** Used to detect table modifications mainly. Return null in case table does not exist. */
- public TableSignature getSignature() throws IOException;
-
- public boolean exists() throws IOException;
-
- public interface TableReader extends Closeable {
-
- /** Move to the next row, return false if no more record. */
- public boolean next() throws IOException;
-
- /** Get the current row. */
- public String[] getRow();
-
- }
-
- // ============================================================================
-
- @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
- public class TableSignature {
-
- @JsonProperty("path")
- private String path;
- @JsonProperty("size")
- private long size;
- @JsonProperty("last_modified_time")
- private long lastModifiedTime;
-
- // for JSON serialization
- public TableSignature() {
- }
-
- public TableSignature(String path, long size, long lastModifiedTime) {
- super();
- this.path = path;
- this.size = size;
- this.lastModifiedTime = lastModifiedTime;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public void setSize(long size) {
- this.size = size;
- }
-
- public void setLastModifiedTime(long lastModifiedTime) {
- this.lastModifiedTime = lastModifiedTime;
- }
-
- public String getPath() {
- return path;
- }
-
- public long getSize() {
- return size;
- }
-
- public long getLastModifiedTime() {
- return lastModifiedTime;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (lastModifiedTime ^ (lastModifiedTime >>> 32));
- result = prime * result + ((path == null) ? 0 : path.hashCode());
- result = prime * result + (int) (size ^ (size >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableSignature other = (TableSignature) obj;
- if (lastModifiedTime != other.lastModifiedTime)
- return false;
- if (path == null) {
- if (other.path != null)
- return false;
- } else if (!path.equals(other.path))
- return false;
- if (size != other.size)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "FileSignature [path=" + path + ", size=" + size + ", lastModifiedTime=" + lastModifiedTime + "]";
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
deleted file mode 100644
index 5f6b664..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- */
-public class SnapshotManager {
-
- private static final Logger logger = LoggerFactory.getLogger(SnapshotManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, SnapshotManager> SERVICE_CACHE = new ConcurrentHashMap<KylinConfig, SnapshotManager>();
-
- public static SnapshotManager getInstance(KylinConfig config) {
- SnapshotManager r = SERVICE_CACHE.get(config);
- if (r == null) {
- r = new SnapshotManager(config);
- SERVICE_CACHE.put(config, r);
- if (SERVICE_CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- }
- return r;
- }
-
- // ============================================================================
-
- private KylinConfig config;
- private ConcurrentHashMap<String, SnapshotTable> snapshotCache; // resource path ==> SnapshotTable
-
- private SnapshotManager(KylinConfig config) {
- this.config = config;
- snapshotCache = new ConcurrentHashMap<String, SnapshotTable>();
- }
-
- public void wipeoutCache() {
- snapshotCache.clear();
- }
-
- public SnapshotTable getSnapshotTable(String resourcePath) throws IOException {
- SnapshotTable r = snapshotCache.get(resourcePath);
- if (r == null) {
- r = load(resourcePath, true);
- snapshotCache.put(resourcePath, r);
- }
- return r;
- }
-
- public void removeSnapshot(String resourcePath) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
- store.deleteResource(resourcePath);
- snapshotCache.remove(resourcePath);
- }
-
- public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
- SnapshotTable snapshot = new SnapshotTable(table);
- snapshot.updateRandomUuid();
-
- String dup = checkDupByInfo(snapshot);
- if (dup != null) {
- logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup);
- return getSnapshotTable(dup);
- }
-
- if (snapshot.getSignature().getSize() / 1024 / 1024 > config.getTableSnapshotMaxMB()) {
- throw new IllegalStateException("Table snapshot should be no greater than " + config.getTableSnapshotMaxMB() //
- + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize());
- }
-
- snapshot.takeSnapshot(table, tableDesc);
-
- return trySaveNewSnapshot(snapshot);
- }
-
- public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException {
-
- String dupTable = checkDupByContent(snapshotTable);
- if (dupTable != null) {
- logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable);
- return getSnapshotTable(dupTable);
- }
-
- save(snapshotTable);
- snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable);
-
- return snapshotTable;
- }
-
- private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
- String resourceDir = snapshot.getResourceDir();
- ArrayList<String> existings = store.listResources(resourceDir);
- if (existings == null)
- return null;
-
- TableSignature sig = snapshot.getSignature();
- for (String existing : existings) {
- SnapshotTable existingTable = load(existing, false); // skip cache,
- // direct load from store
- if (existingTable != null && sig.equals(existingTable.getSignature()))
- return existing;
- }
-
- return null;
- }
-
- private String checkDupByContent(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
- String resourceDir = snapshot.getResourceDir();
- ArrayList<String> existings = store.listResources(resourceDir);
- if (existings == null)
- return null;
-
- for (String existing : existings) {
- SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store
- if (existingTable != null && existingTable.equals(snapshot))
- return existing;
- }
-
- return null;
- }
-
- private void save(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
- String path = snapshot.getResourcePath();
- store.putResource(path, snapshot, SnapshotTableSerializer.FULL_SERIALIZER);
- }
-
- private SnapshotTable load(String resourcePath, boolean loadData) throws IOException {
- logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData);
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
-
- SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);
-
- if (loadData)
- logger.debug("Loaded snapshot at " + resourcePath);
-
- return table;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
deleted file mode 100644
index cc3c637..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.TrieDictionaryBuilder;
-import org.apache.kylin.metadata.model.TableDesc;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class SnapshotTable extends RootPersistentEntity implements ReadableTable {
-
- @JsonProperty("signature")
- private TableSignature signature;
- @JsonProperty("useDictionary")
- private boolean useDictionary;
-
- private List<int[]> rowIndices = Collections.emptyList();
- private Dictionary<String> dict;
-
- // default constructor for JSON serialization
- public SnapshotTable() {
- }
-
- SnapshotTable(ReadableTable table) throws IOException {
- this.signature = table.getSignature();
- this.useDictionary = true;
- }
-
- public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
- this.signature = table.getSignature();
-
- int maxIndex = tableDesc.getMaxColumnIndex();
-
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
-
- TableReader reader = table.getReader();
- while (reader.next()) {
- String[] row = reader.getRow();
- if (row.length <= maxIndex) {
- throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row));
- }
-
- for (String cell : row) {
- if (cell != null)
- b.addValue(cell);
- }
- }
-
- this.dict = b.build(0);
-
- reader = table.getReader();
- ArrayList<int[]> allRowIndices = new ArrayList<int[]>();
- while (reader.next()) {
- String[] row = reader.getRow();
- int[] rowIndex = new int[row.length];
- for (int i = 0; i < row.length; i++) {
- rowIndex[i] = dict.getIdFromValue(row[i]);
- }
- allRowIndices.add(rowIndex);
- }
- this.rowIndices = allRowIndices;
- }
-
- public String getResourcePath() {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName() + "/" + uuid + ".snapshot";
- }
-
- public String getResourceDir() {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName();
- }
-
- @Override
- public TableReader getReader() throws IOException {
- return new TableReader() {
-
- int i = -1;
-
- @Override
- public boolean next() throws IOException {
- i++;
- return i < rowIndices.size();
- }
-
- @Override
- public String[] getRow() {
- int[] rowIndex = rowIndices.get(i);
- String[] row = new String[rowIndex.length];
- for (int x = 0; x < row.length; x++) {
- row[x] = dict.getValueFromId(rowIndex[x]);
- }
- return row;
- }
-
- @Override
- public void close() throws IOException {
- }
- };
- }
-
- @Override
- public TableSignature getSignature() throws IOException {
- return signature;
- }
-
- @Override
- public boolean exists() throws IOException {
- return true;
- }
-
- /**
- * a naive implementation
- *
- * @return
- */
- @Override
- public int hashCode() {
- int[] parts = new int[this.rowIndices.size()];
- for (int i = 0; i < parts.length; ++i)
- parts[i] = Arrays.hashCode(this.rowIndices.get(i));
- return Arrays.hashCode(parts);
- }
-
- @Override
- public boolean equals(Object o) {
- if ((o instanceof SnapshotTable) == false)
- return false;
- SnapshotTable that = (SnapshotTable) o;
-
- //compare row by row
- if (this.rowIndices.size() != that.rowIndices.size())
- return false;
- for (int i = 0; i < this.rowIndices.size(); ++i) {
- if (!ArrayUtils.isEquals(this.rowIndices.get(i), that.rowIndices.get(i)))
- return false;
- }
- return true;
- }
-
- void writeData(DataOutput out) throws IOException {
- out.writeInt(rowIndices.size());
- if (rowIndices.size() > 0) {
- int n = rowIndices.get(0).length;
- out.writeInt(n);
-
- if (this.useDictionary == true) {
- dict.write(out);
- for (int i = 0; i < rowIndices.size(); i++) {
- int[] row = rowIndices.get(i);
- for (int j = 0; j < n; j++) {
- out.writeInt(row[j]);
- }
- }
-
- } else {
- for (int i = 0; i < rowIndices.size(); i++) {
- int[] row = rowIndices.get(i);
- for (int j = 0; j < n; j++) {
- out.writeUTF(dict.getValueFromId(row[j]));
- }
- }
- }
- }
- }
-
- void readData(DataInput in) throws IOException {
- int rowNum = in.readInt();
- if (rowNum > 0) {
- int n = in.readInt();
- rowIndices = new ArrayList<int[]>(rowNum);
-
- if (this.useDictionary == true) {
- this.dict = new TrieDictionary<String>();
- dict.readFields(in);
-
- for (int i = 0; i < rowNum; i++) {
- int[] row = new int[n];
- this.rowIndices.add(row);
- for (int j = 0; j < n; j++) {
- row[j] = in.readInt();
- }
- }
- } else {
- List<String[]> rows = new ArrayList<String[]>(rowNum);
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
-
- for (int i = 0; i < rowNum; i++) {
- String[] row = new String[n];
- rows.add(row);
- for (int j = 0; j < n; j++) {
- row[j] = in.readUTF();
- if (row[j] != null)
- b.addValue(row[j]);
- }
- }
- this.dict = b.build(0);
- for (String[] row : rows) {
- int[] rowIndex = new int[n];
- for (int i = 0; i < n; i++) {
- rowIndex[i] = dict.getIdFromValue(row[i]);
- }
- this.rowIndices.add(rowIndex);
- }
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTableSerializer.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTableSerializer.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTableSerializer.java
deleted file mode 100644
index 0152af6..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTableSerializer.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
-
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotTableSerializer implements Serializer<SnapshotTable> {
-
- public static final SnapshotTableSerializer FULL_SERIALIZER = new SnapshotTableSerializer(false);
- public static final SnapshotTableSerializer INFO_SERIALIZER = new SnapshotTableSerializer(true);
-
- private boolean infoOnly;
-
- SnapshotTableSerializer(boolean infoOnly) {
- this.infoOnly = infoOnly;
- }
-
- @Override
- public void serialize(SnapshotTable obj, DataOutputStream out) throws IOException {
- String json = JsonUtil.writeValueAsIndentString(obj);
- out.writeUTF(json);
-
- if (infoOnly == false)
- obj.writeData(out);
- }
-
- @Override
- public SnapshotTable deserialize(DataInputStream in) throws IOException {
- String json = in.readUTF();
- SnapshotTable obj = JsonUtil.readValue(json, SnapshotTable.class);
-
- if (infoOnly == false)
- obj.readData(in);
-
- return obj;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
deleted file mode 100644
index 5542422..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class DateStrDictionaryTest {
-
- DateStrDictionary dict;
-
- @Before
- public void setup() {
- dict = new DateStrDictionary();
- }
-
- @Test
- public void testMinMaxId() {
- assertEquals(0, dict.getIdFromValue("0000-01-01"));
- assertEquals(DateStrDictionary.ID_9999_12_31, dict.getIdFromValue("9999-12-31"));
-
- try {
- dict.getValueFromId(-2); // -1 is id for NULL
- fail("IllegalArgumentException expected");
- } catch (IllegalArgumentException e) {
- // good
- }
-
- try {
- dict.getValueFromId(DateStrDictionary.ID_9999_12_31 + 1);
- fail("IllegalArgumentException expected");
- } catch (IllegalArgumentException e) {
- // good
- }
-
- try {
- dict.getIdFromValue("10000-1-1");
- fail("IllegalArgumentException expected");
- } catch (IllegalArgumentException e) {
- // good
- }
- }
-
- @Test
- public void testNull() {
- int nullId = dict.getIdFromValue(null);
- assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
- }
-
- @Test
- public void test() {
- checkPair("0001-01-01");
- checkPair("1970-01-02");
- checkPair("1975-06-24");
- checkPair("2024-10-04");
- checkPair("9999-12-31");
- }
-
- @Test
- public void testIllegalArgument() {
- try {
- dict.getIdFromValue("abcd");
- fail("IllegalArgumentException expected");
- } catch (IllegalArgumentException e) {
- // good
- }
-
- try {
- dict.getValueFromId(-2);
- fail("IllegalArgumentException expected");
- } catch (IllegalArgumentException e) {
- // good
- }
- }
-
- private void checkPair(String dateStr) {
- int id = dict.getIdFromValue(dateStr);
- String dateStrBack = dict.getValueFromId(id);
- assertEquals(dateStr, dateStrBack);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
deleted file mode 100644
index 6594f17..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox; It is in the exclude list of default profile in pom.xml
- * @author shaoshi
- *
- */
-public class HiveTableReaderTest extends HBaseMetadataTestCase {
-
- @Test
- public void test() throws IOException {
- HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
- int rowNumber = 0;
- while (reader.next()) {
- String[] row = reader.getRow();
- Assert.assertEquals(9, row.length);
- System.out.println(ArrayUtils.toString(row));
- rowNumber++;
- }
-
- reader.close();
- Assert.assertEquals(10000, rowNumber);
- }
-}
[19/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
deleted file mode 100644
index 0ad7ee1..0000000
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ /dev/null
@@ -1,407 +0,0 @@
-{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "TEST_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
deleted file mode 100644
index 42f4d9c..0000000
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1",
- "name" : "TEST_SELLER_TYPE_DIM",
- "columns" : [ {
- "id" : "1",
- "name" : "SELLER_TYPE_CD",
- "datatype" : "smallint"
- }, {
- "id" : "2",
- "name" : "SELLER_TYPE_DESC",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "GLBL_RPRT_SLR_SGMNT_CD",
- "datatype" : "tinyint"
- }, {
- "id" : "4",
- "name" : "SELLER_GROUP_CD",
- "datatype" : "tinyint"
- }, {
- "id" : "5",
- "name" : "SELLER_GROUP_DESC",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
deleted file mode 100644
index d451e34..0000000
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6",
- "name" : "TEST_SITES",
- "columns" : [ {
- "id" : "1",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "2",
- "name" : "SITE_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_DOMAIN_CODE",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "DFAULT_LSTG_CURNCY",
- "datatype" : "int"
- }, {
- "id" : "5",
- "name" : "EOA_EMAIL_CSTMZBL_SITE_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "6",
- "name" : "SITE_CNTRY_ID",
- "datatype" : "int"
- }, {
- "id" : "7",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/16337839-a90f-46fa-a67b-37e330fe9685.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/16337839-a90f-46fa-a67b-37e330fe9685.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/16337839-a90f-46fa-a67b-37e330fe9685.snapshot
deleted file mode 100644
index d6b66f0..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/16337839-a90f-46fa-a67b-37e330fe9685.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot
deleted file mode 100644
index fab45f7..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/66de9855-8815-4f3b-8ac6-c9386db0b196.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/66de9855-8815-4f3b-8ac6-c9386db0b196.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/66de9855-8815-4f3b-8ac6-c9386db0b196.snapshot
deleted file mode 100644
index 8ce1150..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/66de9855-8815-4f3b-8ac6-c9386db0b196.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot
deleted file mode 100644
index 5e78425..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/9a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/9a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/9a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot
deleted file mode 100644
index f06e2df..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/9a1fbc0d-8790-4f32-99e1-7c60a9d124d4.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot
deleted file mode 100644
index d3ee8c7..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e5f004b5-0be1-4ccf-8950-20f8f86c1999.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e6cbec7f-0326-443d-b45f-54549d5609c6.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e6cbec7f-0326-443d-b45f-54549d5609c6.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e6cbec7f-0326-443d-b45f-54549d5609c6.snapshot
deleted file mode 100644
index 9b46b06..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/e6cbec7f-0326-443d-b45f-54549d5609c6.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/f91cd3a0-3ae5-4306-87ab-1624d45a6cde.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/f91cd3a0-3ae5-4306-87ab-1624d45a6cde.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/f91cd3a0-3ae5-4306-87ab-1624d45a6cde.snapshot
deleted file mode 100644
index 2c6ba58..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CAL_DT.csv/f91cd3a0-3ae5-4306-87ab-1624d45a6cde.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/59a675fc-64d2-40d1-bbd7-492db2510cd1.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/59a675fc-64d2-40d1-bbd7-492db2510cd1.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/59a675fc-64d2-40d1-bbd7-492db2510cd1.snapshot
deleted file mode 100644
index c3046c3..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/59a675fc-64d2-40d1-bbd7-492db2510cd1.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot
deleted file mode 100644
index eed588c..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/89715f33-15c9-4745-83f9-f2b9817d9100.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/df8a90f1-bbbe-49ff-937a-2a2a6e5f8615.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/df8a90f1-bbbe-49ff-937a-2a2a6e5f8615.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/df8a90f1-bbbe-49ff-937a-2a2a6e5f8615.snapshot
deleted file mode 100644
index 9a4ff3d..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/df8a90f1-bbbe-49ff-937a-2a2a6e5f8615.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot
deleted file mode 100644
index bdc1afa..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot
deleted file mode 100644
index 482c778..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/97238b77-a45d-420f-a249-5f1572ea4d62.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/97238b77-a45d-420f-a249-5f1572ea4d62.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/97238b77-a45d-420f-a249-5f1572ea4d62.snapshot
deleted file mode 100644
index d9a04d6..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/97238b77-a45d-420f-a249-5f1572ea4d62.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot
deleted file mode 100644
index d4bc695..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/b43dd3f1-9a34-4721-8abc-df90962e94d8.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/d1b6a60a-14d4-4653-92f7-c96d01191c75.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/d1b6a60a-14d4-4653-92f7-c96d01191c75.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/d1b6a60a-14d4-4653-92f7-c96d01191c75.snapshot
deleted file mode 100644
index 07d7965..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/d1b6a60a-14d4-4653-92f7-c96d01191c75.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/f818054b-1a76-4cbd-bdd8-7871a240360c.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/f818054b-1a76-4cbd-bdd8-7871a240360c.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/f818054b-1a76-4cbd-bdd8-7871a240360c.snapshot
deleted file mode 100644
index 140962f..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SELLER_TYPE_DIM.csv/f818054b-1a76-4cbd-bdd8-7871a240360c.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1271e1f2-dd65-4249-a1f2-aca02a651158.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1271e1f2-dd65-4249-a1f2-aca02a651158.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1271e1f2-dd65-4249-a1f2-aca02a651158.snapshot
deleted file mode 100644
index 8b405f4..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1271e1f2-dd65-4249-a1f2-aca02a651158.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1888015f-f059-4ccc-8324-0cdd70b46673.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1888015f-f059-4ccc-8324-0cdd70b46673.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1888015f-f059-4ccc-8324-0cdd70b46673.snapshot
deleted file mode 100644
index 2fbd6c5..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/1888015f-f059-4ccc-8324-0cdd70b46673.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot
deleted file mode 100644
index ef5c7fd..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot
deleted file mode 100644
index ce75b97..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/7d70b82e-43f6-4999-b012-0e91a9bb8408.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/d95e0b5b-a399-4f08-8f40-b0fa53b565e0.snapshot
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/d95e0b5b-a399-4f08-8f40-b0fa53b565e0.snapshot b/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/d95e0b5b-a399-4f08-8f40-b0fa53b565e0.snapshot
deleted file mode 100644
index 19fe6a5..0000000
Binary files a/examples/test_case_data/localmeta/table_snapshot/TEST_SITES.csv/d95e0b5b-a399-4f08-8f40-b0fa53b565e0.snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/minicluster/hbase-export.tar.gz
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/hbase-export.tar.gz b/examples/test_case_data/minicluster/hbase-export.tar.gz
deleted file mode 100755
index 5800a56..0000000
Binary files a/examples/test_case_data/minicluster/hbase-export.tar.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
deleted file mode 100644
index 06f9870..0000000
--- a/examples/test_case_data/minicluster/kylin.properties
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=ADMIN:KYLIN@localhost
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/tmp
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Hive JDBC connection properties; Not in use now
-kylin.hive.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-kylin.hive.jdbc.url=jdbc:hive2:///
-kylin.hive.jdbc.user=
-kylin.hive.jdbc.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
deleted file mode 100644
index 81704f2..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
deleted file mode 100644
index 81704f2..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/minicluster/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.xml b/examples/test_case_data/minicluster/kylin_job_conf.xml
deleted file mode 100644
index 81704f2..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. See accompanying LICENSE file.
--->
-
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
deleted file mode 100644
index 7cb985c..0000000
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.5</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.resource-calculator</name>
- <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.capacity</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-am-resource-percent</name>
- <value>0.5</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>1</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
deleted file mode 100644
index 9aa588c..0000000
--- a/examples/test_case_data/sandbox/core-site.xml
+++ /dev/null
@@ -1,165 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
- </property>
-
- <property>
- <name>fs.trash.interval</name>
- <value>360</value>
- </property>
-
- <property>
- <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
- <value>120</value>
- </property>
-
- <property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.falcon.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hbase.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hcat.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hive.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hue.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hue.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.oozie.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.root.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.root.hosts</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.security.auth_to_local</name>
- <value>DEFAULT</value>
- </property>
-
- <property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
- </property>
-
- <property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
- </property>
-
- <property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
- </property>
-
- <property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
- </property>
-
- <property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
- </property>
-
- <property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
- </property>
-
- <property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
- </property>
-
- <property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
- </property>
-
- <property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml
deleted file mode 100644
index a590979..0000000
--- a/examples/test_case_data/sandbox/hadoop-policy.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>security.admin.operations.protocol.acl</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>security.client.datanode.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.datanode.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.inter.datanode.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.inter.tracker.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.job.client.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.job.task.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.namenode.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.refresh.policy.protocol.acl</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>security.refresh.usertogroups.mappings.protocol.acl</name>
- <value>hadoop</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml
deleted file mode 100644
index 178b559..0000000
--- a/examples/test_case_data/sandbox/hbase-policy.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>security.admin.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
- </property>
-
- <property>
- <name>security.masterregion.protocol.acl</name>
- <value>*</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
deleted file mode 100644
index 734908e..0000000
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ /dev/null
@@ -1,194 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- </property>
-
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
- </property>
-
- <property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
- <property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
- </property>
-
- <property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
- </property>
-
- <property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hbase.hstore.blockingStoreFiles</name>
- <value>10</value>
- </property>
-
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- </property>
-
- <property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
- </property>
-
- <property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
- </property>
-
- <property>
- <name>hbase.master.port</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- </property>
-
- <property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
- </property>
-
- <property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
- </property>
-
- <property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
- </property>
-
- <property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
- </property>
-
- <property>
- <name>hbase.security.authentication</name>
- <value>simple</value>
- </property>
-
- <property>
- <name>hbase.security.authorization</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hbase.superuser</name>
- <value>hbase</value>
- </property>
-
- <property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
- </property>
-
- <property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
- </property>
-
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
deleted file mode 100644
index 1175fff..0000000
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ /dev/null
@@ -1,285 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- </property>
-
- <property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
- </property>
-
- <property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
- </property>
-
- <property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>dfs.https.port</name>
- <value>50470</value>
- </property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- </property>
-
- <property>
- <name>dfs.journalnode.https-address</name>
- <value>0.0.0.0:8481</value>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- </property>
-
- <property>
- <name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
- </property>
-
- <property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
- </property>
-
- <property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
- </property>
-
- <property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
- </property>
-
- <property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
- </property>
-
- <property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
- </property>
-
- <property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>1</value>
- </property>
-
- <property>
- <name>dfs.replication.max</name>
- <value>50</value>
- </property>
-
- <property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.webhdfs.enabled</name>
- <value>true</value>
- <final>true</final>
- </property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
deleted file mode 100644
index f4c7738..0000000
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ /dev/null
@@ -1,794 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>ambari.hive.db.schema.name</name>
- <value>hive</value>
- </property>
-
- <property>
- <name>datanucleus.cache.level2.type</name>
- <value>none</value>
- </property>
-
- <property>
- <name>hive.auto.convert.join</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.auto.convert.join.noconditionaltask</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>1000000000</value>
- </property>
-
- <property>
- <name>hive.auto.convert.sortmerge.join</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.cbo.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.cli.print.header</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.cluster.delegation.token.store.class</name>
- <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value>
- </property>
-
- <property>
- <name>hive.cluster.delegation.token.store.zookeeper.connectString</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
- <property>
- <name>hive.cluster.delegation.token.store.zookeeper.znode</name>
- <value>/hive/cluster/delegation</value>
- </property>
-
- <property>
- <name>hive.compactor.abortedtxn.threshold</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>hive.compactor.check.interval</name>
- <value>300s</value>
- </property>
-
- <property>
- <name>hive.compactor.delta.num.threshold</name>
- <value>10</value>
- </property>
-
- <property>
- <name>hive.compactor.delta.pct.threshold</name>
- <value>0.1f</value>
- </property>
-
- <property>
- <name>hive.compactor.initiator.on</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.compactor.worker.threads</name>
- <value>0</value>
- </property>
-
- <property>
- <name>hive.compactor.worker.timeout</name>
- <value>86400s</value>
- </property>
-
- <property>
- <name>hive.compute.query.using.stats</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.conf.restricted.list</name>
- <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value>
- </property>
-
- <property>
- <name>hive.convert.join.bucket.mapjoin.tez</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.enforce.bucketing</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.enforce.sorting</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.enforce.sortmergebucketmapjoin</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.exec.compress.intermediate</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.exec.compress.output</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.exec.dynamic.partition</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.exec.dynamic.partition.mode</name>
- <value>nonstrict</value>
- </property>
-
- <property>
- <name>hive.exec.failure.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
- <name>hive.exec.max.created.files</name>
- <value>100000</value>
- </property>
-
- <property>
- <name>hive.exec.max.dynamic.partitions</name>
- <value>5000</value>
- </property>
-
- <property>
- <name>hive.exec.max.dynamic.partitions.pernode</name>
- <value>2000</value>
- </property>
-
- <property>
- <name>hive.exec.orc.compression.strategy</name>
- <value>SPEED</value>
- </property>
-
- <property>
- <name>hive.exec.orc.default.compress</name>
- <value>ZLIB</value>
- </property>
-
- <property>
- <name>hive.exec.orc.default.stripe.size</name>
- <value>67108864</value>
- </property>
-
- <property>
- <name>hive.exec.parallel</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.exec.parallel.thread.number</name>
- <value>8</value>
- </property>
-
- <property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
- </property>
-
- <property>
- <name>hive.exec.reducers.bytes.per.reducer</name>
- <value>67108864</value>
- </property>
-
- <property>
- <name>hive.exec.reducers.max</name>
- <value>1009</value>
- </property>
-
- <property>
- <name>hive.exec.scratchdir</name>
- <value>/tmp/hive</value>
- </property>
-
- <property>
- <name>hive.exec.submit.local.task.via.child</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.exec.submitviachild</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.execution.engine</name>
- <value>mr</value>
- </property>
-
- <property>
- <name>hive.fetch.task.aggr</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.fetch.task.conversion</name>
- <value>more</value>
- </property>
-
- <property>
- <name>hive.fetch.task.conversion.threshold</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>hive.heapsize</name>
- <value>250</value>
- </property>
-
- <property>
- <name>hive.limit.optimize.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.limit.pushdown.memory.usage</name>
- <value>0.04</value>
- </property>
-
- <property>
- <name>hive.map.aggr</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.map.aggr.hash.force.flush.memory.threshold</name>
- <value>0.9</value>
- </property>
-
- <property>
- <name>hive.map.aggr.hash.min.reduction</name>
- <value>0.5</value>
- </property>
-
- <property>
- <name>hive.map.aggr.hash.percentmemory</name>
- <value>0.5</value>
- </property>
-
- <property>
- <name>hive.mapjoin.bucket.cache.size</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>hive.mapjoin.optimized.hashtable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.mapred.reduce.tasks.speculative.execution</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.merge.mapfiles</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.merge.mapredfiles</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.merge.orcfile.stripe.level</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.merge.rcfile.block.level</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.merge.size.per.task</name>
- <value>256000000</value>
- </property>
-
- <property>
- <name>hive.merge.smallfiles.avgsize</name>
- <value>16000000</value>
- </property>
-
- <property>
- <name>hive.merge.tezfiles</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.metastore.authorization.storage.checks</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.metastore.cache.pinobjtypes</name>
- <value>Table,Database,Type,FieldSchema,Order</value>
- </property>
-
- <property>
- <name>hive.metastore.client.connect.retry.delay</name>
- <value>5s</value>
- </property>
-
- <property>
- <name>hive.metastore.client.socket.timeout</name>
- <value>1800s</value>
- </property>
-
- <property>
- <name>hive.metastore.connect.retries</name>
- <value>24</value>
- </property>
-
- <property>
- <name>hive.metastore.execute.setugi</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.metastore.failure.retries</name>
- <value>24</value>
- </property>
-
- <property>
- <name>hive.metastore.kerberos.keytab.file</name>
- <value>/etc/security/keytabs/hive.service.keytab</value>
- </property>
-
- <property>
- <name>hive.metastore.kerberos.principal</name>
- <value>hive/_HOST@EXAMPLE.COM</value>
- </property>
-
- <property>
- <name>hive.metastore.pre.event.listeners</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value>
- </property>
-
- <property>
- <name>hive.metastore.sasl.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.metastore.server.max.threads</name>
- <value>100000</value>
- </property>
-
- <property>
- <name>hive.metastore.uris</name>
- <value>thrift://sandbox.hortonworks.com:9083</value>
- </property>
-
- <property>
- <name>hive.metastore.warehouse.dir</name>
- <value>/apps/hive/warehouse</value>
- </property>
-
- <property>
- <name>hive.optimize.bucketmapjoin</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.bucketmapjoin.sortedmerge</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.optimize.constant.propagation</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.index.filter</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.metadataonly</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.null.scan</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.reducededuplication</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.optimize.reducededuplication.min.reducer</name>
- <value>4</value>
- </property>
-
- <property>
- <name>hive.optimize.sort.dynamic.partition</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.orc.compute.splits.num.threads</name>
- <value>10</value>
- </property>
-
- <property>
- <name>hive.orc.splits.include.file.footer</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.prewarm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.prewarm.numcontainers</name>
- <value>10</value>
- </property>
-
- <property>
- <name>hive.security.authenticator.manager</name>
- <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value>
- </property>
-
- <property>
- <name>hive.security.authorization.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.security.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value>
- </property>
-
- <property>
- <name>hive.security.metastore.authenticator.manager</name>
- <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value>
- </property>
-
- <property>
- <name>hive.security.metastore.authorization.auth.reads</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
- </property>
-
- <property>
- <name>hive.server2.allow.user.substitution</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.server2.authentication</name>
- <value>NONE</value>
- </property>
-
- <property>
- <name>hive.server2.authentication.spnego.keytab</name>
- <value>HTTP/_HOST@EXAMPLE.COM</value>
- </property>
-
- <property>
- <name>hive.server2.authentication.spnego.principal</name>
- <value>/etc/security/keytabs/spnego.service.keytab</value>
- </property>
-
- <property>
- <name>hive.server2.enable.doAs</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.server2.enable.impersonation</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.server2.logging.operation.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.server2.logging.operation.log.location</name>
- <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
- </property>
-
- <property>
- <name>hive.server2.support.dynamic.service.discovery</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.server2.table.type.mapping</name>
- <value>CLASSIC</value>
- </property>
-
- <property>
- <name>hive.server2.tez.default.queues</name>
- <value>default</value>
- </property>
-
- <property>
- <name>hive.server2.tez.initialize.default.sessions</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.server2.tez.sessions.per.default.queue</name>
- <value>1</value>
- </property>
-
- <property>
- <name>hive.server2.thrift.http.path</name>
- <value>cliservice</value>
- </property>
-
- <property>
- <name>hive.server2.thrift.http.port</name>
- <value>10001</value>
- </property>
-
- <property>
- <name>hive.server2.thrift.max.worker.threads</name>
- <value>500</value>
- </property>
-
- <property>
- <name>hive.server2.thrift.port</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>hive.server2.thrift.sasl.qop</name>
- <value>auth</value>
- </property>
-
- <property>
- <name>hive.server2.transport.mode</name>
- <value>binary</value>
- </property>
-
- <property>
- <name>hive.server2.use.SSL</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.server2.zookeeper.namespace</name>
- <value>hiveserver2</value>
- </property>
-
- <property>
- <name>hive.smbjoin.cache.rows</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>hive.stats.autogather</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.stats.dbclass</name>
- <value>fs</value>
- </property>
-
- <property>
- <name>hive.stats.fetch.column.stats</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.stats.fetch.partition.stats</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.support.concurrency</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.tez.auto.reducer.parallelism</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.tez.container.size</name>
- <value>250</value>
- </property>
-
- <property>
- <name>hive.tez.cpu.vcores</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>hive.tez.dynamic.partition.pruning</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.tez.dynamic.partition.pruning.max.data.size</name>
- <value>104857600</value>
- </property>
-
- <property>
- <name>hive.tez.dynamic.partition.pruning.max.event.size</name>
- <value>1048576</value>
- </property>
-
- <property>
- <name>hive.tez.input.format</name>
- <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value>
- </property>
-
- <property>
- <name>hive.tez.java.opts</name>
- <value>-server -Xmx200m -Djava.net.preferIPv4Stack=true</value>
- </property>
-
- <property>
- <name>hive.tez.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>hive.tez.max.partition.factor</name>
- <value>2.0</value>
- </property>
-
- <property>
- <name>hive.tez.min.partition.factor</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>hive.tez.smb.number.waves</name>
- <value>0.5</value>
- </property>
-
- <property>
- <name>hive.txn.manager</name>
- <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
- </property>
-
- <property>
- <name>hive.txn.max.open.batch</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>hive.txn.timeout</name>
- <value>300</value>
- </property>
-
- <property>
- <name>hive.user.install.directory</name>
- <value>/user/</value>
- </property>
-
- <property>
- <name>hive.users.in.admin.role</name>
- <value>hue,hive</value>
- </property>
-
- <property>
- <name>hive.vectorized.execution.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.vectorized.execution.reduce.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hive.vectorized.groupby.checkinterval</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>hive.vectorized.groupby.flush.percent</name>
- <value>0.1</value>
- </property>
-
- <property>
- <name>hive.vectorized.groupby.maxentries</name>
- <value>100000</value>
- </property>
-
- <property>
- <name>hive.zookeeper.client.port</name>
- <value>2181</value>
- </property>
-
- <property>
- <name>hive.zookeeper.namespace</name>
- <value>hive_zookeeper_namespace</value>
- </property>
-
- <property>
- <name>hive.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
- <property>
- <name>hive_metastore_user_passwd</name>
- <value>hive</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>com.mysql.jdbc.Driver</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>hive</value>
- </property>
-
-</configuration>
[13/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
deleted file mode 100644
index c62477e..0000000
--- a/job/pom.xml
+++ /dev/null
@@ -1,257 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <artifactId>kylin-job</artifactId>
- <name>Kylin:Job</name>
- <url>http://maven.apache.org</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.parent.version}</version>
- </dependency>
-
- <!--Kylin Jar -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-cube</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-daemon</groupId>
- <artifactId>commons-daemon</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-email</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- Env & Test -->
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <classifier>hadoop2</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-model</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>${hive-hcatalog.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase-hadoop2.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
-
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <minimizeJar>false</minimizeJar>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>job</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java b/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
deleted file mode 100644
index 87c4705..0000000
--- a/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.hadoop.hive.IJoinedFlatTableDesc;
-
-public abstract class AbstractJobBuilder {
-
- protected static final String JOB_WORKING_DIR_PREFIX = "kylin-";
-
- protected JobEngineConfig engineConfig;
- protected String submitter;
-
- public AbstractJobBuilder(JobEngineConfig engineConfig) {
- this.engineConfig = engineConfig;
- }
-
- public AbstractJobBuilder setSubmitter(String submitter) {
- this.submitter = submitter;
- return this;
- }
-
- public String getSubmitter() {
- return submitter;
- }
-
- protected StringBuilder appendExecCmdParameters(StringBuilder cmd, String paraName, String paraValue) {
- return cmd.append(" -").append(paraName).append(" ").append(paraValue);
- }
-
- // return in full-qualified name, that is "dbname.tablename"
- protected String getIntermediateHiveTableName(IJoinedFlatTableDesc intermediateTableDesc, String jobUuid) {
- return engineConfig.getConfig().getHiveDatabaseForIntermediateTable() + "." + intermediateTableDesc.getTableName(jobUuid);
- }
-
- protected String getIntermediateHiveTableLocation(IJoinedFlatTableDesc intermediateTableDesc, String jobUUID) {
- return getJobWorkingDir(jobUUID) + "/" + intermediateTableDesc.getTableName(jobUUID);
- }
-
- protected AbstractExecutable createIntermediateHiveTableStep(IJoinedFlatTableDesc intermediateTableDesc, String jobId) {
-
- final String useDatabaseHql = "USE " + engineConfig.getConfig().getHiveDatabaseForIntermediateTable() + ";";
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobId);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, getJobWorkingDir(jobId), jobId);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobId, this.engineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive ");
- buf.append(" -e \"");
- buf.append(useDatabaseHql + "\n");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
-
- return step;
- }
-
- protected String getJobWorkingDir(String uuid) {
- return engineConfig.getHdfsWorkingDirectory() + JOB_WORKING_DIR_PREFIX + uuid;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
deleted file mode 100644
index 0a08709..0000000
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ /dev/null
@@ -1,781 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeDescUpgrader;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.v1.CubeInstance;
-import org.apache.kylin.cube.model.v1.CubeSegment;
-import org.apache.kylin.cube.model.v1.CubeSegmentStatusEnum;
-import org.apache.kylin.cube.model.v1.CubeStatusEnum;
-import org.apache.kylin.job.common.HadoopShellExecutable;
-import org.apache.kylin.job.common.MapReduceExecutable;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.BaseCuboidJob;
-import org.apache.kylin.job.hadoop.cube.CubeHFileJob;
-import org.apache.kylin.job.hadoop.cube.FactDistinctColumnsJob;
-import org.apache.kylin.job.hadoop.cube.MergeCuboidJob;
-import org.apache.kylin.job.hadoop.cube.NDCuboidJob;
-import org.apache.kylin.job.hadoop.cube.RangeKeyDistributionJob;
-import org.apache.kylin.job.hadoop.dict.CreateDictionaryJob;
-import org.apache.kylin.job.hadoop.hbase.BulkLoadJob;
-import org.apache.kylin.job.hadoop.hbase.CreateHTableJob;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-/**
- * This is the utility class to migrate the Kylin metadata format from v1 to v2;
- *
- * @author shaoshi
- */
-public class CubeMetadataUpgrade {
-
- private KylinConfig config = null;
- private ResourceStore store;
-
- private List<String> updatedResources = Lists.newArrayList();
- private List<String> errorMsgs = Lists.newArrayList();
-
- private static final Log logger = LogFactory.getLog(CubeMetadataUpgrade.class);
-
- public CubeMetadataUpgrade(String newMetadataUrl) {
- KylinConfig.destoryInstance();
- System.setProperty(KylinConfig.KYLIN_CONF, newMetadataUrl);
- KylinConfig.getInstanceFromEnv().setMetadataUrl(newMetadataUrl);
-
- config = KylinConfig.getInstanceFromEnv();
- store = getStore();
- }
-
- public void upgrade() {
-
- upgradeTableDesc();
- upgradeTableDesceExd();
- upgradeCubeDesc();
- upgradeProjectInstance();
- upgradeCubeInstance();
- upgradeJobInstance();
- copyDictionaryForFK();
- verify();
-
- }
-
- public void cleanup() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.getInstance(config);
- CubeManager cubeManager = CubeManager.getInstance(config);
-
- List<String> activeResourceList = Lists.newArrayList();
- for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
- for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
- activeResourceList.addAll(segment.getSnapshotPaths());
- activeResourceList.addAll(segment.getDictionaryPaths());
- }
- }
-
- List<String> toDeleteResource = Lists.newArrayList();
- List<String> activeResource = Lists.newArrayList();
- try {
- ArrayList<String> snapshotTables = getStore().listResources(ResourceStore.SNAPSHOT_RESOURCE_ROOT);
-
- for (String snapshotTable : snapshotTables) {
- ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
- if (snapshotNames != null)
- for (String snapshot : snapshotNames) {
- if (!activeResourceList.contains(snapshot)) {
- toDeleteResource.add(snapshot);
-
- } else {
- activeResource.add(snapshot);
- }
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- try {
- ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
-
- for (String table : dictTables) {
- ArrayList<String> tableColNames = getStore().listResources(table);
- if (tableColNames != null)
- for (String tableCol : tableColNames) {
- ArrayList<String> dictionaries = getStore().listResources(tableCol);
- if (dictionaries != null)
- for (String dict : dictionaries)
- if (!activeResourceList.contains(dict)) {
- toDeleteResource.add(dict);
- } else {
- activeResource.add(dict);
- }
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- if (toDeleteResource.size() > 0) {
- logger.info("The following resources is never needed, will be dropped, number :" + toDeleteResource.size());
-
- for (String s : toDeleteResource) {
- logger.info(s);
- try {
- getStore().deleteResource(s);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- }
-
- public void verify() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
- CubeDescManager.getInstance(config);
- CubeManager.getInstance(config);
- ProjectManager.getInstance(config);
- //cleanup();
-
- }
-
- private List<String> listResourceStore(String pathRoot) {
- List<String> paths = null;
- try {
- paths = store.collectResourceRecursively(pathRoot, MetadataConstants.FILE_SURFIX);
- } catch (IOException e1) {
- e1.printStackTrace();
- errorMsgs.add("Get IOException when scan resource store at: " + ResourceStore.CUBE_DESC_RESOURCE_ROOT);
- }
-
- return paths;
- }
-
- private void upgradeCubeDesc() {
- logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-
- List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
- for (String path : paths) {
-
- try {
- CubeDescUpgrader upgrade = new CubeDescUpgrader(path);
- CubeDesc ndesc = upgrade.upgrade();
- ndesc.setSignature(ndesc.calculateSignature());
-
- getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER);
- getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
- updatedResources.add(ndesc.getResourcePath());
- } catch (IOException e) {
- e.printStackTrace();
- errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
- }
- }
-
- }
-
- private void upgradeTableDesc() {
- List<String> paths = listResourceStore(ResourceStore.TABLE_RESOURCE_ROOT);
- for (String path : paths) {
- TableDesc t;
- try {
- t = store.getResource(path, TableDesc.class, MetadataManager.TABLE_SERIALIZER);
- t.init();
-
- // if it only has 1 "." in the path, delete the old resource if it exists
- if (path.substring(path.indexOf(".")).length() == MetadataConstants.FILE_SURFIX.length()) {
- getStore().deleteResource(path);
- // the new source will be new;
- t.setLastModified(0);
- getStore().putResource(t.getResourcePath(), t, MetadataManager.TABLE_SERIALIZER);
- updatedResources.add(t.getResourcePath());
- }
- } catch (IOException e) {
- e.printStackTrace();
- errorMsgs.add("Upgrade TableDesc at '" + path + "' failed: " + e.getLocalizedMessage());
- }
-
- }
-
- }
-
- @SuppressWarnings("unchecked")
- private void upgradeTableDesceExd() {
-
- List<String> paths = listResourceStore(ResourceStore.TABLE_EXD_RESOURCE_ROOT);
- for (String path : paths) {
- Map<String, String> attrs = Maps.newHashMap();
-
- InputStream is = null;
- try {
- is = store.getResource(path).inputStream;
- if (is == null) {
- continue;
- }
- try {
- attrs.putAll(JsonUtil.readValue(is, HashMap.class));
- } finally {
- if (is != null)
- is.close();
- }
- } catch (IOException e) {
- e.printStackTrace();
- errorMsgs.add("Upgrade TableDescExd at '" + path + "' failed: " + e.getLocalizedMessage());
- }
-
- // parse table identity from file name
- String file = path;
- if (file.indexOf("/") > -1) {
- file = file.substring(file.lastIndexOf("/") + 1);
- }
- String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase();
-
- // for metadata upgrade, convert resource path to new pattern (<DB>.<TABLE>.json)
- if (tableIdentity.indexOf(".") < 0) {
- tableIdentity = appendDBName(tableIdentity);
- try {
- getMetadataManager().saveTableExd(tableIdentity, attrs);
- //delete old resoruce if it exists;
- getStore().deleteResource(path);
- updatedResources.add(path);
- } catch (IOException e) {
- e.printStackTrace();
- errorMsgs.add("Upgrade TableDescExd at '" + path + "' failed: " + e.getLocalizedMessage());
- }
-
- }
-
- }
-
- }
-
- public String appendDBName(String table) {
-
- if (table.indexOf(".") > 0)
- return table;
-
- Map<String, TableDesc> map = this.getMetadataManager().getAllTablesMap();
-
- int count = 0;
- String result = null;
- for (TableDesc t : map.values()) {
- if (t.getName().equalsIgnoreCase(table)) {
- result = t.getIdentity();
- count++;
- }
- }
-
- if (count == 1)
- return result;
-
- if (count > 1) {
- errorMsgs.add("There are more than 1 table named with '" + table + "' in different database; ");
- }
-
- if (count == 0) {
- errorMsgs.add("No table definition for '" + table + "'; any project, cube refers it should remove the reference;");
- }
-
- return null;
- }
-
- private void upgradeProjectInstance() {
- List<String> paths = listResourceStore(ResourceStore.PROJECT_RESOURCE_ROOT);
- for (String path : paths) {
- try {
- org.apache.kylin.cube.model.v1.ProjectInstance oldPrj = store.getResource(path, org.apache.kylin.cube.model.v1.ProjectInstance.class, new JsonSerializer<org.apache.kylin.cube.model.v1.ProjectInstance>(org.apache.kylin.cube.model.v1.ProjectInstance.class));
-
- ProjectInstance newPrj = new ProjectInstance();
- newPrj.setUuid(oldPrj.getUuid());
- newPrj.setName(oldPrj.getName());
- newPrj.setOwner(oldPrj.getOwner());
- newPrj.setDescription(oldPrj.getDescription());
- newPrj.setLastModified(oldPrj.getLastModified());
- newPrj.setCreateTimeUTC(RootPersistentEntity.parseTime(oldPrj.getCreateTime()));
- newPrj.setStatus(oldPrj.getStatus());
- List<RealizationEntry> realizationEntries = Lists.newArrayList();
- for (String cube : oldPrj.getCubes()) {
- RealizationEntry entry = new RealizationEntry();
- entry.setType(RealizationType.CUBE);
- entry.setRealization(cube);
- realizationEntries.add(entry);
- }
- newPrj.setRealizationEntries(realizationEntries);
-
- Set<String> tables = Sets.newHashSet();
- for (String table : oldPrj.getTables()) {
- String tb = this.appendDBName(table);
- if (tb != null)
- tables.add(this.appendDBName(tb));
- }
- newPrj.setTables(tables);
-
- store.putResource(newPrj.getResourcePath(), newPrj, ProjectManager.PROJECT_SERIALIZER);
- updatedResources.add(path);
- } catch (IOException e) {
- e.printStackTrace();
- errorMsgs.add("Upgrade Project at '" + path + "' failed: " + e.getLocalizedMessage());
- }
- }
-
- }
-
- private void upgradeCubeInstance() {
-
- ResourceStore store = getStore();
- List<String> paths = listResourceStore(ResourceStore.CUBE_RESOURCE_ROOT);
- for (String path : paths) {
-
- CubeInstance cubeInstance = null;
- try {
- cubeInstance = store.getResource(path, CubeInstance.class, new JsonSerializer<CubeInstance>(CubeInstance.class));
- cubeInstance.setConfig(config);
-
- org.apache.kylin.cube.CubeInstance newInstance = new org.apache.kylin.cube.CubeInstance();
- newInstance.setName(cubeInstance.getName());
- newInstance.setDescName(cubeInstance.getDescName());
- newInstance.setOwner(cubeInstance.getOwner());
- newInstance.setUuid(cubeInstance.getUuid());
- newInstance.setVersion(cubeInstance.getVersion());
- newInstance.setCreateTimeUTC(RootPersistentEntity.parseTime(cubeInstance.getCreateTime()));
- newInstance.setLastModified(cubeInstance.getLastModified());
-
- //status
- if (cubeInstance.getStatus() == CubeStatusEnum.BUILDING) {
- newInstance.setStatus(RealizationStatusEnum.BUILDING);
- } else if (cubeInstance.getStatus() == CubeStatusEnum.DESCBROKEN) {
- newInstance.setStatus(RealizationStatusEnum.DESCBROKEN);
- } else if (cubeInstance.getStatus() == CubeStatusEnum.DISABLED) {
- newInstance.setStatus(RealizationStatusEnum.DISABLED);
- } else if (cubeInstance.getStatus() == CubeStatusEnum.READY) {
- newInstance.setStatus(RealizationStatusEnum.READY);
- }
-
- List<org.apache.kylin.cube.CubeSegment> newSegments = Lists.newArrayList();
- // segment
- for (CubeSegment segment : cubeInstance.getSegments()) {
- org.apache.kylin.cube.CubeSegment newSeg = new org.apache.kylin.cube.CubeSegment();
- newSegments.add(newSeg);
-
- newSeg.setUuid(segment.getUuid());
- newSeg.setName(segment.getName());
- newSeg.setStorageLocationIdentifier(segment.getStorageLocationIdentifier());
- newSeg.setDateRangeStart(segment.getDateRangeStart());
- newSeg.setDateRangeEnd(segment.getDateRangeEnd());
-
- if (segment.getStatus() == CubeSegmentStatusEnum.NEW) {
- newSeg.setStatus(SegmentStatusEnum.NEW);
- } else if (segment.getStatus() == CubeSegmentStatusEnum.READY) {
- newSeg.setStatus(SegmentStatusEnum.READY);
- } else if (segment.getStatus() == CubeSegmentStatusEnum.READY_PENDING) {
- newSeg.setStatus(SegmentStatusEnum.READY_PENDING);
- }
-
- newSeg.setSizeKB(segment.getSizeKB());
- newSeg.setInputRecords(segment.getSourceRecords());
- newSeg.setInputRecordsSize(segment.getSourceRecordsSize());
- newSeg.setLastBuildTime(segment.getLastBuildTime());
- newSeg.setLastBuildJobID(segment.getLastBuildJobID());
- newSeg.setCreateTimeUTC(RootPersistentEntity.parseTime(segment.getCreateTime()));
- newSeg.setBinarySignature(segment.getBinarySignature());
-
- ConcurrentHashMap<String, String> newDictionaries = new ConcurrentHashMap<String, String>();
-
- for (Map.Entry<String, String> e : segment.getDictionaries().entrySet()) {
- String key = e.getKey();
- String[] tableCol = StringUtils.split(key, "/");
- key = appendDBName(tableCol[0]) + "/" + tableCol[1];
- newDictionaries.put(key, e.getValue());
- }
- newSeg.setDictionaries(newDictionaries);
-
- ConcurrentHashMap<String, String> newSnapshots = new ConcurrentHashMap<String, String>();
-
- for (Map.Entry<String, String> e : segment.getSnapshots().entrySet()) {
- newSnapshots.put(appendDBName(e.getKey()), e.getValue());
- }
- newSeg.setSnapshots(newSnapshots);
- }
-
- newInstance.setSegments(newSegments);
- store.putResource(newInstance.getResourcePath(), newInstance, CubeManager.CUBE_SERIALIZER);
- } catch (Exception e) {
- logger.error("Error during load cube instance " + path, e);
- }
- }
- }
-
- private void copyDictionaryForFK() {
- CubeManager cubeManager = CubeManager.getInstance(config);
- List<org.apache.kylin.cube.CubeInstance> cubeInstances = cubeManager.listAllCubes();
-
- Set<String> changedCubes = Sets.newHashSet();
- for (org.apache.kylin.cube.CubeInstance newInstance : cubeInstances) {
-
- boolean updated = false;
- DataModelDesc dataModelDesc = null;
- try {
- String modelName = this.getCubeDescManager().getCubeDesc(newInstance.getDescName()).getModelName();
- dataModelDesc = this.getMetadataManager().getDataModelDesc(modelName);
- Map<String, String> pkToFK = Maps.newHashMap();
- for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
- if (lookupDesc.getJoin() != null) {
- JoinDesc join = lookupDesc.getJoin();
- for (int i = 0; i < join.getForeignKey().length; i++) {
- pkToFK.put(lookupDesc.getTable() + "/" + join.getPrimaryKey()[i], dataModelDesc.getFactTable() + "/" + join.getForeignKey()[i]);
- }
- }
- }
-
- List<Pair<String, String>> newDictionaries = Lists.newArrayList();
-
- // segment
- for (org.apache.kylin.cube.CubeSegment newSeg : newInstance.getSegments()) {
-
- for (Map.Entry<String, String> e : newSeg.getDictionaries().entrySet()) {
- String key = e.getKey();
- if (pkToFK.containsKey(key) && !newSeg.getDictionaries().containsKey(pkToFK.get(key))) {
- logger.debug("Duplicate dictionary for FK " + pkToFK.get(key) + " in cube " + newInstance.getName());
- changedCubes.add(newInstance.getName());
- newDictionaries.add(new Pair<String, String>(pkToFK.get(key), e.getValue()));
-
- }
- }
- for (Pair<String, String> dict : newDictionaries) {
- newSeg.getDictionaries().put(dict.getFirst(), dict.getSecond());
- updated = true;
- }
- }
-
- if (updated)
- store.putResource(newInstance.getResourcePath(), newInstance, CubeManager.CUBE_SERIALIZER);
- } catch (Exception e) {
- logger.error("Error during upgrade cube instance " + newInstance.getName(), e);
- }
- }
-
- if (changedCubes.size() > 0)
- logger.info("Updated these cubeInstances: " + changedCubes);
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private CubeDescManager getCubeDescManager() {
- return CubeDescManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(config);
- }
-
- private ExecutableManager getExecutableManager() {
- return ExecutableManager.getInstance(config);
- }
-
- private void upgradeJobInstance() {
- try {
- List<String> paths = getStore().collectResourceRecursively(ResourceStore.JOB_PATH_ROOT, "");
- for (String path : paths) {
- upgradeJobInstance(path);
- }
-
- for (String folder : new String[] { ResourceStore.JOB_PATH_ROOT, ResourceStore.JOB_OUTPUT_PATH_ROOT }) {
- for (String res : getStore().listResources(folder)) {
- getStore().deleteResource(res);
- }
- getStore().deleteResource(folder);
- }
- } catch (IOException ex) {
- errorMsgs.add("upgrade job failed" + ex.getLocalizedMessage());
- throw new RuntimeException(ex);
- }
-
- }
-
- private ExecutableState parseState(JobStatusEnum state) {
- switch (state) {
- case NEW:
- case PENDING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case ERROR:
- return ExecutableState.ERROR;
- case DISCARDED:
- return ExecutableState.DISCARDED;
- default:
- return ExecutableState.DISCARDED;
- }
- }
-
- private ExecutableState parseState(JobStepStatusEnum state) {
- switch (state) {
- case NEW:
- case PENDING:
- case WAITING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case ERROR:
- return ExecutableState.ERROR;
- case DISCARDED:
- return ExecutableState.DISCARDED;
- default:
- return ExecutableState.DISCARDED;
- }
-
- }
-
- private void upgradeJobInstance(String path) throws IOException {
- JobInstance job = getStore().getResource(path, JobInstance.class, new JsonSerializer<JobInstance>(JobInstance.class));
- long lastModified = job.getLastModified();
- if (System.currentTimeMillis() - lastModified > 2592000000l) {
- // old than 30 days, skip;
- return;
- }
- CubingJob cubingJob = new CubingJob();
- cubingJob.setId(job.getId());
- cubingJob.setName(job.getName());
- cubingJob.setCubeName(job.getRelatedCube());
- cubingJob.setSubmitter(job.getSubmitter());
- for (JobInstance.JobStep step : job.getSteps()) {
- final AbstractExecutable executable = parseToExecutable(step);
- cubingJob.addTask(executable);
- }
- getExecutableManager().addJob(cubingJob);
-
- cubingJob.setStartTime(job.getExecStartTime());
- cubingJob.setEndTime(job.getExecEndTime());
- cubingJob.setMapReduceWaitTime(job.getMrWaiting());
- getExecutableManager().resetJobOutput(cubingJob.getId(), parseState(job.getStatus()), job.getStatus().toString());
-
- for (int i = 0, size = job.getSteps().size(); i < size; ++i) {
- final JobInstance.JobStep jobStep = job.getSteps().get(i);
- final String outputPath = ResourceStore.JOB_OUTPUT_PATH_ROOT + "/" + job.getId() + "." + i;
- final InputStream inputStream = getStore().getResource(outputPath).inputStream;
-
- String output = null;
- if (inputStream != null) {
- @SuppressWarnings("unchecked")
- HashMap<String, String> job_output = JsonUtil.readValue(inputStream, HashMap.class);
-
- if (job_output != null) {
- output = job_output.get("output");
- }
- org.apache.commons.io.IOUtils.closeQuietly(inputStream);
- }
- updateJobStepOutput(jobStep, output, cubingJob.getTasks().get(i));
- }
- }
-
- private void updateJobStepOutput(JobInstance.JobStep step, String output, AbstractExecutable task) {
- task.setStartTime(step.getExecStartTime());
- task.setEndTime(step.getExecEndTime());
- if (task instanceof MapReduceExecutable) {
- ((MapReduceExecutable) task).setMapReduceWaitTime(step.getExecWaitTime() * 1000);
- }
- getExecutableManager().resetJobOutput(task.getId(), parseState(step.getStatus()), output);
- }
-
- private AbstractExecutable parseToExecutable(JobInstance.JobStep step) {
- AbstractExecutable result;
- switch (step.getCmdType()) {
- case SHELL_CMD_HADOOP: {
- ShellExecutable executable = new ShellExecutable();
- executable.setCmd(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_FACTDISTINCT: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(FactDistinctColumnsJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_BASECUBOID: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(BaseCuboidJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_NDCUBOID: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(NDCuboidJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_RANGEKEYDISTRIBUTION: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(RangeKeyDistributionJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_CONVERTHFILE: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(CubeHFileJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_MERGECUBOID: {
- MapReduceExecutable executable = new MapReduceExecutable();
- executable.setMapReduceJobClass(MergeCuboidJob.class);
- executable.setMapReduceParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_NO_MR_DICTIONARY: {
- HadoopShellExecutable executable = new HadoopShellExecutable();
- executable.setName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY);
- executable.setJobClass(CreateDictionaryJob.class);
- executable.setJobParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADDOP_NO_MR_CREATEHTABLE: {
- HadoopShellExecutable executable = new HadoopShellExecutable();
- executable.setJobClass(CreateHTableJob.class);
- executable.setJobParams(step.getExecCmd());
- result = executable;
- break;
- }
- case JAVA_CMD_HADOOP_NO_MR_BULKLOAD: {
- HadoopShellExecutable executable = new HadoopShellExecutable();
- executable.setJobClass(BulkLoadJob.class);
- executable.setJobParams(step.getExecCmd());
- result = executable;
- break;
- }
- default:
- throw new RuntimeException("invalid step type:" + step.getCmdType());
- }
- result.setName(step.getName());
- return result;
- }
-
- public static void main(String[] args) {
-
- if (!(args != null && (args.length == 1 || args.length == 2))) {
- System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
- return;
- }
-
- String exportFolder = args[0];
- boolean verify = false;
- if (args.length == 2 && "verify".equals(args[1])) {
- System.out.println("Only verify the metadata in folder " + exportFolder);
- verify = true;
- }
-
- CubeMetadataUpgrade instance = null;
- if (verify) {
- instance = new CubeMetadataUpgrade(exportFolder);
- instance.verify();
- instance.copyDictionaryForFK();
- } else {
- File oldMetaFolder = new File(exportFolder);
- if (!oldMetaFolder.exists()) {
- System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
- return;
- }
-
- if (!oldMetaFolder.isDirectory()) {
- System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
- return;
- }
-
- String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
- try {
- FileUtils.deleteDirectory(new File(newMetadataUrl));
- FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- instance = new CubeMetadataUpgrade(newMetadataUrl);
- instance.upgrade();
- logger.info("=================================================================");
- logger.info("Run CubeMetadataUpgrade completed;");
-
- }
-
- logger.info("=================================================================");
- if (instance.errorMsgs.size() > 0) {
- logger.info("Here are the error/warning messages, you may need check:");
- for (String s : instance.errorMsgs) {
- logger.warn(s);
- }
- } else {
- logger.info("No error or warning messages; The migration is success.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/JobInstance.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JobInstance.java b/job/src/main/java/org/apache/kylin/job/JobInstance.java
deleted file mode 100644
index 82d4753..0000000
--- a/job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ /dev/null
@@ -1,499 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepCmdTypeEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.engine.JobEngineConfig;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Lists;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JobInstance extends RootPersistentEntity implements Comparable<JobInstance> {
-
- public static final String JOB_WORKING_DIR_PREFIX = "kylin-";
-
- public static final String YARN_APP_ID = "yarn_application_id";
- public static final String YARN_APP_URL = "yarn_application_tracking_url";
- public static final String MR_JOB_ID = "mr_job_id";
- public static final String HDFS_BYTES_WRITTEN = "hdfs_bytes_written";
- public static final String SOURCE_RECORDS_COUNT = "source_records_count";
- public static final String SOURCE_RECORDS_SIZE = "source_records_size";
-
- public static String getStepIdentity(JobInstance jobInstance, JobStep jobStep) {
- return jobInstance.getRelatedCube() + "." + jobInstance.getUuid() + "." + jobStep.getSequenceID();
- }
-
- public static String getJobIdentity(JobInstance jobInstance) {
- return jobInstance.getRelatedCube() + "." + jobInstance.getUuid();
- }
-
- public static String getJobWorkingDir(JobInstance jobInstance, JobEngineConfig engineConfig) {
- return getJobWorkingDir(jobInstance.getUuid(), engineConfig.getHdfsWorkingDirectory());
- }
-
- public static String getJobWorkingDir(String jobUuid, String hdfsWorkdingDir) {
- if (jobUuid == null || jobUuid.equals("")) {
- throw new IllegalArgumentException("jobUuid can't be null or empty");
- }
- return hdfsWorkdingDir + JOB_WORKING_DIR_PREFIX + jobUuid;
- }
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("type")
- private CubeBuildTypeEnum type; // java implementation
- @JsonProperty("duration")
- private long duration;
- @JsonProperty("related_cube")
- private String relatedCube;
- @JsonProperty("related_segment")
- private String relatedSegment;
- @JsonProperty("exec_start_time")
- private long execStartTime;
- @JsonProperty("exec_end_time")
- private long execEndTime;
- @JsonProperty("mr_waiting")
- private long mrWaiting = 0;
- @JsonManagedReference
- @JsonProperty("steps")
- private List<JobStep> steps;
- @JsonProperty("submitter")
- private String submitter;
- @JsonProperty("job_status")
- private JobStatusEnum status;
-
- public JobStep getRunningStep() {
- for (JobStep step : this.getSteps()) {
- if (step.getStatus().equals(JobStepStatusEnum.RUNNING) || step.getStatus().equals(JobStepStatusEnum.WAITING)) {
- return step;
- }
- }
-
- return null;
- }
-
- @JsonProperty("progress")
- public double getProgress() {
- int completedStepCount = 0;
- for (JobStep step : this.getSteps()) {
- if (step.getStatus().equals(JobStepStatusEnum.FINISHED)) {
- completedStepCount++;
- }
- }
-
- return 100.0 * completedStepCount / steps.size();
- }
-
- public JobStatusEnum getStatus() {
- return this.status;
- }
-
- public void setStatus(JobStatusEnum status) {
- this.status = status;
- }
-
- // @JsonProperty("job_status")
- // public JobStatusEnum getStatus() {
- //
- // // JobStatusEnum finalJobStatus;
- // int compositResult = 0;
- //
- // // if steps status are all NEW, then job status is NEW
- // // if steps status are all FINISHED, then job status is FINISHED
- // // if steps status are all PENDING, then job status is PENDING
- // // if steps status are FINISHED and PENDING, the job status is PENDING
- // // if one of steps status is RUNNING, then job status is RUNNING
- // // if one of steps status is ERROR, then job status is ERROR
- // // if one of steps status is KILLED, then job status is KILLED
- // // default status is RUNNING
- //
- // System.out.println(this.getName());
- //
- // for (JobStep step : this.getSteps()) {
- // //System.out.println("step: " + step.getSequenceID() + "'s status:" + step.getStatus());
- // compositResult = compositResult | step.getStatus().getCode();
- // }
- //
- // System.out.println();
- //
- // if (compositResult == JobStatusEnum.FINISHED.getCode()) {
- // return JobStatusEnum.FINISHED;
- // } else if (compositResult == JobStatusEnum.NEW.getCode()) {
- // return JobStatusEnum.NEW;
- // } else if (compositResult == JobStatusEnum.PENDING.getCode()) {
- // return JobStatusEnum.PENDING;
- // } else if (compositResult == (JobStatusEnum.FINISHED.getCode() | JobStatusEnum.PENDING.getCode())) {
- // return JobStatusEnum.PENDING;
- // } else if ((compositResult & JobStatusEnum.ERROR.getCode()) == JobStatusEnum.ERROR.getCode()) {
- // return JobStatusEnum.ERROR;
- // } else if ((compositResult & JobStatusEnum.DISCARDED.getCode()) == JobStatusEnum.DISCARDED.getCode()) {
- // return JobStatusEnum.DISCARDED;
- // } else if ((compositResult & JobStatusEnum.RUNNING.getCode()) == JobStatusEnum.RUNNING.getCode()) {
- // return JobStatusEnum.RUNNING;
- // }
- //
- // return JobStatusEnum.RUNNING;
- // }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public CubeBuildTypeEnum getType() {
- return type;
- }
-
- public void setType(CubeBuildTypeEnum type) {
- this.type = type;
- }
-
- public long getDuration() {
- return duration;
- }
-
- public void setDuration(long duration) {
- this.duration = duration;
- }
-
- public String getRelatedCube() {
- return relatedCube;
- }
-
- public void setRelatedCube(String relatedCube) {
- this.relatedCube = relatedCube;
- }
-
- public String getRelatedSegment() {
- return relatedSegment;
- }
-
- public void setRelatedSegment(String relatedSegment) {
- this.relatedSegment = relatedSegment;
- }
-
- /**
- * @return the execStartTime
- */
- public long getExecStartTime() {
- return execStartTime;
- }
-
- /**
- * @param execStartTime the execStartTime to set
- */
- public void setExecStartTime(long execStartTime) {
- this.execStartTime = execStartTime;
- }
-
- /**
- * @return the execEndTime
- */
- public long getExecEndTime() {
- return execEndTime;
- }
-
- /**
- * @param execEndTime the execEndTime to set
- */
- public void setExecEndTime(long execEndTime) {
- this.execEndTime = execEndTime;
- }
-
- public long getMrWaiting() {
- return this.mrWaiting;
- }
-
- public void setMrWaiting(long mrWaiting) {
- this.mrWaiting = mrWaiting;
- }
-
- public List<JobStep> getSteps() {
- if (steps == null) {
- steps = Lists.newArrayList();
- }
- return steps;
- }
-
- public void clearSteps() {
- getSteps().clear();
- }
-
- public void addSteps(Collection<JobStep> steps) {
- this.getSteps().addAll(steps);
- }
-
- public void addStep(JobStep step) {
- getSteps().add(step);
- }
-
- public void addStep(int index, JobStep step) {
- getSteps().add(index, step);
- }
-
- public JobStep findStep(String stepName) {
- for (JobStep step : getSteps()) {
- if (stepName.equals(step.getName())) {
- return step;
- }
- }
- return null;
- }
-
- public String getSubmitter() {
- return submitter;
- }
-
- public void setSubmitter(String submitter) {
- this.submitter = submitter;
- }
-
- @JsonIgnoreProperties(ignoreUnknown = true)
- public static class JobStep implements Comparable<JobStep> {
-
- @JsonBackReference
- private JobInstance jobInstance;
-
- @JsonProperty("id")
- private String id;
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("sequence_id")
- private int sequenceID;
-
- @JsonProperty("exec_cmd")
- private String execCmd;
-
- @JsonProperty("interrupt_cmd")
- private String InterruptCmd;
-
- @JsonProperty("exec_start_time")
- private long execStartTime;
- @JsonProperty("exec_end_time")
- private long execEndTime;
- @JsonProperty("exec_wait_time")
- private long execWaitTime;
-
- @JsonProperty("step_status")
- private JobStepStatusEnum status;
-
- @JsonProperty("cmd_type")
- private JobStepCmdTypeEnum cmdType = JobStepCmdTypeEnum.SHELL_CMD_HADOOP;
-
- @JsonProperty("info")
- private ConcurrentHashMap<String, String> info = new ConcurrentHashMap<String, String>();
-
- @JsonProperty("run_async")
- private boolean runAsync = false;
-
- private ConcurrentHashMap<String, String> getInfo() {
- return info;
- }
-
- public void putInfo(String key, String value) {
- getInfo().put(key, value);
- }
-
- public String getInfo(String key) {
- return getInfo().get(key);
- }
-
- public void clearInfo() {
- getInfo().clear();
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public int getSequenceID() {
- return sequenceID;
- }
-
- public void setSequenceID(int sequenceID) {
- this.sequenceID = sequenceID;
- }
-
- public String getExecCmd() {
- return execCmd;
- }
-
- public void setExecCmd(String execCmd) {
- this.execCmd = execCmd;
- }
-
- public JobStepStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(JobStepStatusEnum status) {
- this.status = status;
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * @return the execStartTime
- */
- public long getExecStartTime() {
- return execStartTime;
- }
-
- /**
- * @param execStartTime the execStartTime to set
- */
- public void setExecStartTime(long execStartTime) {
- this.execStartTime = execStartTime;
- }
-
- /**
- * @return the execEndTime
- */
- public long getExecEndTime() {
- return execEndTime;
- }
-
- /**
- * @param execEndTime the execEndTime to set
- */
- public void setExecEndTime(long execEndTime) {
- this.execEndTime = execEndTime;
- }
-
- public long getExecWaitTime() {
- return execWaitTime;
- }
-
- public void setExecWaitTime(long execWaitTime) {
- this.execWaitTime = execWaitTime;
- }
-
- public String getInterruptCmd() {
- return InterruptCmd;
- }
-
- public void setInterruptCmd(String interruptCmd) {
- InterruptCmd = interruptCmd;
- }
-
- public JobStepCmdTypeEnum getCmdType() {
- return cmdType;
- }
-
- public void setCmdType(JobStepCmdTypeEnum cmdType) {
- this.cmdType = cmdType;
- }
-
- /**
- * @return the runAsync
- */
- public boolean isRunAsync() {
- return runAsync;
- }
-
- /**
- * @param runAsync the runAsync to set
- */
- public void setRunAsync(boolean runAsync) {
- this.runAsync = runAsync;
- }
-
- /**
- * @return the jobInstance
- */
- public JobInstance getJobInstance() {
- return jobInstance;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + sequenceID;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- JobStep other = (JobStep) obj;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (sequenceID != other.sequenceID)
- return false;
- return true;
- }
-
- @Override
- public int compareTo(JobStep o) {
- if (this.sequenceID < o.sequenceID) {
- return -1;
- } else if (this.sequenceID > o.sequenceID) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
- @Override
- public int compareTo(JobInstance o) {
- return o.lastModified < this.lastModified ? -1 : o.lastModified > this.lastModified ? 1 : 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
deleted file mode 100644
index eb6d27b..0000000
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
-import org.apache.kylin.job.hadoop.hive.IJoinedFlatTableDesc;
-import org.apache.kylin.job.hadoop.hive.IntermediateColumnDesc;
-import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class JoinedFlatTable {
-
- public static String generateCreateTableStatement(IJoinedFlatTableDesc intermediateTableDesc, String storageDfsDir, String jobUUID) {
- StringBuilder ddl = new StringBuilder();
-
- ddl.append("CREATE EXTERNAL TABLE IF NOT EXISTS " + intermediateTableDesc.getTableName(jobUUID) + "\n");
-
- ddl.append("(" + "\n");
- for (int i = 0; i < intermediateTableDesc.getColumnList().size(); i++) {
- IntermediateColumnDesc col = intermediateTableDesc.getColumnList().get(i);
- if (i > 0) {
- ddl.append(",");
- }
- ddl.append(colName(col.getCanonicalName()) + " " + SqlHiveDataTypeMapping.getHiveDataType(col.getDataType()) + "\n");
- }
- ddl.append(")" + "\n");
-
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
- ddl.append("STORED AS SEQUENCEFILE" + "\n");
- ddl.append("LOCATION '" + storageDfsDir + "/" + intermediateTableDesc.getTableName(jobUUID) + "';").append("\n");
- // ddl.append("TBLPROPERTIES ('serialization.null.format'='\\\\N')" +
- // ";\n");
- return ddl.toString();
- }
-
- public static String generateDropTableStatement(IJoinedFlatTableDesc intermediateTableDesc, String jobUUID) {
- StringBuilder ddl = new StringBuilder();
- ddl.append("DROP TABLE IF EXISTS " + intermediateTableDesc.getTableName(jobUUID) + ";").append("\n");
- return ddl.toString();
- }
-
- public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, String jobUUID, JobEngineConfig engineConfig) throws IOException {
- StringBuilder sql = new StringBuilder();
-
- File hadoopPropertiesFile = new File(engineConfig.getHadoopJobConfFilePath(intermediateTableDesc.getCapacity()));
-
- if (hadoopPropertiesFile.exists()) {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- try {
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
- NodeList nl = doc.getElementsByTagName("property");
- for (int i = 0; i < nl.getLength(); i++) {
- String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
- String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
- if (name.equals("tmpjars") == false) {
- sql.append("SET " + name + "=" + value + ";").append("\n");
- }
- }
-
- } catch (ParserConfigurationException e) {
- throw new IOException(e);
- } catch (SAXException e) {
- throw new IOException(e);
- }
- }
-
- // hard coded below mr parameters to enable map-side join
- sql.append("SET hive.exec.compress.output=true;").append("\n");
- sql.append("SET hive.auto.convert.join.noconditionaltask = true;").append("\n");
- sql.append("SET hive.auto.convert.join.noconditionaltask.size = 300000000;").append("\n");
- sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName(jobUUID) + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
-
- return sql.toString();
- }
-
- public static String generateSelectDataStatement(IJoinedFlatTableDesc intermediateTableDesc) {
- StringBuilder sql = new StringBuilder();
- sql.append("SELECT" + "\n");
- String tableAlias;
- Map<String, String> tableAliasMap = buildTableAliasMap(intermediateTableDesc.getDataModel());
- for (int i = 0; i < intermediateTableDesc.getColumnList().size(); i++) {
- IntermediateColumnDesc col = intermediateTableDesc.getColumnList().get(i);
- if (i > 0) {
- sql.append(",");
- }
- tableAlias = tableAliasMap.get(col.getTableName());
- sql.append(tableAlias + "." + col.getColumnName() + "\n");
- }
- appendJoinStatement(intermediateTableDesc, sql, tableAliasMap);
- appendWhereStatement(intermediateTableDesc, sql, tableAliasMap);
- return sql.toString();
- }
-
- private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) {
- Map<String, String> tableAliasMap = new HashMap<String, String>();
-
- addTableAlias(dataModelDesc.getFactTable(), tableAliasMap);
-
- for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
- JoinDesc join = lookupDesc.getJoin();
- if (join != null) {
- addTableAlias(lookupDesc.getTable(), tableAliasMap);
- }
- }
- return tableAliasMap;
- }
-
- // The table alias used to be "FACT_TABLE" and "LOOKUP_#", but that's too unpredictable
- // for those who want to write a filter. (KYLIN-900)
- // Also yet don't support joining the same table more than once, since table name is the map key.
- private static void addTableAlias(String table, Map<String, String> tableAliasMap) {
- String alias;
- int cut = table.lastIndexOf('.');
- if (cut < 0)
- alias = table;
- else
- alias = table.substring(cut + 1);
-
- tableAliasMap.put(table, alias);
- }
-
- private static void appendJoinStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
- Set<String> dimTableCache = new HashSet<String>();
-
- DataModelDesc dataModelDesc = intermediateTableDesc.getDataModel();
- String factTableName = dataModelDesc.getFactTable();
- String factTableAlias = tableAliasMap.get(factTableName);
- sql.append("FROM " + factTableName + " as " + factTableAlias + " \n");
-
- for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
- JoinDesc join = lookupDesc.getJoin();
- if (join != null && join.getType().equals("") == false) {
- String joinType = join.getType().toUpperCase();
- String dimTableName = lookupDesc.getTable();
- if (!dimTableCache.contains(dimTableName)) {
- TblColRef[] pk = join.getPrimaryKeyColumns();
- TblColRef[] fk = join.getForeignKeyColumns();
- if (pk.length != fk.length) {
- throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
- }
- sql.append(joinType + " JOIN " + dimTableName + " as " + tableAliasMap.get(dimTableName) + "\n");
- sql.append("ON ");
- for (int i = 0; i < pk.length; i++) {
- if (i > 0) {
- sql.append(" AND ");
- }
- sql.append(factTableAlias + "." + fk[i].getName() + " = " + tableAliasMap.get(dimTableName) + "." + pk[i].getName());
- }
- sql.append("\n");
-
- dimTableCache.add(dimTableName);
- }
- }
- }
- }
-
- private static void appendWhereStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
- if (!(intermediateTableDesc instanceof CubeJoinedFlatTableDesc)) {
- return;//TODO: for now only cube segments support filter and partition
- }
- CubeJoinedFlatTableDesc desc = (CubeJoinedFlatTableDesc) intermediateTableDesc;
-
- boolean hasCondition = false;
- StringBuilder whereBuilder = new StringBuilder();
- whereBuilder.append("WHERE");
-
- CubeDesc cubeDesc = desc.getCubeDesc();
- DataModelDesc model = cubeDesc.getModel();
-
- if (model.getFilterCondition() != null && model.getFilterCondition().equals("") == false) {
- whereBuilder.append(" (").append(model.getFilterCondition()).append(") ");
- hasCondition = true;
- }
-
- CubeSegment cubeSegment = desc.getCubeSegment();
-
- if (null != cubeSegment) {
- PartitionDesc partDesc = model.getPartitionDesc();
- long dateStart = cubeSegment.getDateRangeStart();
- long dateEnd = cubeSegment.getDateRangeEnd();
-
- if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
- whereBuilder.append(hasCondition ? " AND (" : " (");
- whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd, tableAliasMap));
- whereBuilder.append(")\n");
- hasCondition = true;
- }
- }
-
- if (hasCondition) {
- sql.append(whereBuilder.toString());
- }
- }
-
- private static String colName(String canonicalColName) {
- return canonicalColName.replace(".", "_");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/Scheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/Scheduler.java b/job/src/main/java/org/apache/kylin/job/Scheduler.java
deleted file mode 100644
index cc2dff9..0000000
--- a/job/src/main/java/org/apache/kylin/job/Scheduler.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.common.lock.JobLock;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.SchedulerException;
-import org.apache.kylin.job.execution.Executable;
-
-/**
- * Created by qianzhou on 12/15/14.
- */
-public interface Scheduler<T extends Executable> {
-
- void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException;
-
- void shutdown() throws SchedulerException;
-
- boolean stop(T executable) throws SchedulerException;
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java b/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
deleted file mode 100644
index 27b8e87..0000000
--- a/job/src/main/java/org/apache/kylin/job/cmd/BaseCommandOutput.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cmd;
-
-/**
- * Created by qianzhou on 12/4/14.
- */
-public abstract class BaseCommandOutput implements ICommandOutput {
-
- @Override
- public void log(String message) {
- this.appendOutput(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cmd/ICommandOutput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/ICommandOutput.java b/job/src/main/java/org/apache/kylin/job/cmd/ICommandOutput.java
deleted file mode 100644
index 7dfdb13..0000000
--- a/job/src/main/java/org/apache/kylin/job/cmd/ICommandOutput.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cmd;
-
-import org.apache.kylin.common.util.Logger;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-
-/**
- * @author xjiang
- *
- */
-public interface ICommandOutput extends Logger {
-
- public void setStatus(JobStepStatusEnum status);
-
- public JobStepStatusEnum getStatus();
-
- public void appendOutput(String message);
-
- public String getOutput();
-
- public void setExitCode(int exitCode);
-
- public int getExitCode();
-
- public void reset();
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cmd/IJobCommand.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/IJobCommand.java b/job/src/main/java/org/apache/kylin/job/cmd/IJobCommand.java
deleted file mode 100644
index 4aaf4b8..0000000
--- a/job/src/main/java/org/apache/kylin/job/cmd/IJobCommand.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cmd;
-
-import org.apache.kylin.job.exception.JobException;
-
-/**
- * @author xjiang
- *
- */
-public interface IJobCommand {
-
- public ICommandOutput execute() throws JobException;
-
- public void cancel() throws JobException;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/cmd/ShellCmd.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cmd/ShellCmd.java b/job/src/main/java/org/apache/kylin/job/cmd/ShellCmd.java
deleted file mode 100644
index 9ad35cb..0000000
--- a/job/src/main/java/org/apache/kylin/job/cmd/ShellCmd.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.cmd;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.FutureTask;
-
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.exception.JobException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xjiang
- *
- */
-public class ShellCmd implements IJobCommand {
-
- private static Logger log = LoggerFactory.getLogger(ShellCmd.class);
-
- private final String executeCommand;
- private final ICommandOutput output;
- private final boolean isAsync;
- private final CliCommandExecutor cliCommandExecutor;
-
- private FutureTask<Integer> future;
-
- protected ShellCmd(String executeCmd, ICommandOutput out, String host, String user, String password, boolean async) {
- this.executeCommand = executeCmd;
- this.output = out;
- cliCommandExecutor = new CliCommandExecutor();
- cliCommandExecutor.setRunAtRemote(host, user, password);
- this.isAsync = async;
- }
-
- public ShellCmd(String executeCmd, String host, String user, String password, boolean async) {
- this(executeCmd, new ShellCmdOutput(), host, user, password, async);
- }
-
- @Override
- public ICommandOutput execute() throws JobException {
-
- final ExecutorService executor = Executors.newSingleThreadExecutor();
- future = new FutureTask<Integer>(new Callable<Integer>() {
- public Integer call() throws JobException, IOException {
- executor.shutdown();
- return executeCommand(executeCommand);
- }
- });
- executor.execute(future);
-
- int exitCode = -1;
- if (!isAsync) {
- try {
- exitCode = future.get();
- log.info("finish executing");
- } catch (CancellationException e) {
- log.debug("Command is cancelled");
- exitCode = -2;
- } catch (Exception e) {
- throw new JobException("Error when execute job " + executeCommand, e);
- } finally {
- if (exitCode == 0) {
- output.setStatus(JobStepStatusEnum.FINISHED);
- } else if (exitCode == -2) {
- output.setStatus(JobStepStatusEnum.DISCARDED);
- } else {
- output.setStatus(JobStepStatusEnum.ERROR);
- }
- output.setExitCode(exitCode);
- }
- }
- return output;
- }
-
- protected int executeCommand(String command) throws JobException, IOException {
- output.reset();
- output.setStatus(JobStepStatusEnum.RUNNING);
- return cliCommandExecutor.execute(command, output).getFirst();
- }
-
- @Override
- public void cancel() throws JobException {
- future.cancel(true);
- }
-
- public static void main(String[] args) throws JobException {
- ShellCmdOutput output = new ShellCmdOutput();
- ShellCmd shellCmd = new ShellCmd(args[0], output, args[1], args[2], args[3], false);
- shellCmd.execute();
-
- System.out.println("============================================================================");
- System.out.println(output.getExitCode());
- System.out.println(output.getOutput());
- }
-}
[34/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
deleted file mode 100644
index ce86c66..0000000
--- a/dictionary/pom.xml
+++ /dev/null
@@ -1,128 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-dictionary</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:Dictionary</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <properties>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.parent.version}</version>
- </dependency>
-
- <!--Kylin Jar -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-metadata</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <!-- Env & Test -->
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <!-- protobuf version conflict with hbase-->
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>${hive-hcatalog.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase-hadoop2.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/BytesConverter.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/BytesConverter.java b/dictionary/src/main/java/org/apache/kylin/dict/BytesConverter.java
deleted file mode 100644
index 1ae2dc1..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/BytesConverter.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-public interface BytesConverter<T> {
-
- public byte[] convertToBytes(T v);
-
- public T convertFromBytes(byte[] b, int offset, int length);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
deleted file mode 100644
index 14086c6..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.Date;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Dictionary;
-
-/**
- * A dictionary for date string (date only, no time).
- *
- * Dates are numbered from 0000-1-1 -- 0 for "0000-1-1", 1 for "0000-1-2", 2 for "0000-1-3" and
- * up to 3652426 for "9999-12-31".
- *
- * Note the implementation is not thread-safe.
- *
- * @author yangli9
- */
-public class DateStrDictionary extends Dictionary<String> {
-
- static final int ID_9999_12_31 = 3652426; // assume 0 based
-
- // ============================================================================
-
- private String pattern;
- private int baseId;
- private int maxId;
-
- public DateStrDictionary() {
- init(DateFormat.DEFAULT_DATE_PATTERN, 0);
- }
-
- public DateStrDictionary(String datePattern, int baseId) {
- init(datePattern, baseId);
- }
-
- private void init(String datePattern, int baseId) {
- this.pattern = datePattern;
- this.baseId = baseId;
- this.maxId = baseId + ID_9999_12_31;
- }
-
- @Override
- public int getMinId() {
- return baseId;
- }
-
- @Override
- public int getMaxId() {
- return maxId;
- }
-
- @Override
- public int getSizeOfId() {
- return 3;
- }
-
- @Override
- public int getSizeOfValue() {
- return pattern.length();
- }
-
- @Override
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null || len == 0;
- }
-
- @Override
- final protected int getIdFromValueImpl(String value, int roundFlag) {
- Date date = DateFormat.stringToDate(value, pattern);
- int id = calcIdFromSeqNo(getNumOfDaysSince0000(date));
- if (id < baseId || id > maxId)
- throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]");
-
- return id;
- }
-
- @Override
- final protected String getValueFromIdImpl(int id) {
- if (id < baseId || id > maxId)
- throw new IllegalArgumentException("ID '" + id + "' is out of range [" + baseId + "," + maxId + "]");
- Date d = getDateFromNumOfDaysSince0000(calcSeqNoFromId(id));
- return DateFormat.dateToString(d, pattern);
- }
-
- private int getNumOfDaysSince0000(Date d) {
- // 86400000 = 1000 * 60 * 60 * 24
- // -719530 is offset of 0000-01-01
- return (int) (d.getTime() / 86400000 + 719530);
- }
-
- private Date getDateFromNumOfDaysSince0000(int n) {
- long millis = ((long) n - 719530) * 86400000;
- return new Date(millis);
- }
-
- @Override
- final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- try {
- return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- @Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- String date = getValueFromId(id);
- byte bytes[];
- try {
- bytes = date.getBytes("ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
- return bytes.length;
- }
-
- private int calcIdFromSeqNo(int seq) {
- return seq < 0 ? seq : baseId + seq;
- }
-
- private int calcSeqNoFromId(int id) {
- return id - baseId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(pattern);
- out.writeInt(baseId);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- String pattern = in.readUTF();
- int baseId = in.readInt();
- init(pattern, baseId);
- }
-
- @Override
- public int hashCode() {
- return 31 * baseId + pattern.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DateStrDictionary))
- return false;
- DateStrDictionary that = (DateStrDictionary) o;
- return StringUtils.equals(this.pattern, that.pattern) && this.baseId == that.baseId;
- }
-
- @Override
- public void dump(PrintStream out) {
- out.println(this.toString());
- }
-
- @Override
- public String toString() {
- return "DateStrDictionary [pattern=" + pattern + ", baseId=" + baseId + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
deleted file mode 100644
index 0ba6566..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.*;
-import org.apache.kylin.dict.lookup.ReadableTable;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class DictionaryGenerator {
-
- private static final int DICT_MAX_CARDINALITY = getDictionaryMaxCardinality();
-
- private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
-
- private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
-
- private static int getDictionaryMaxCardinality() {
- try {
- return KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
- } catch (Throwable e) {
- return 5000000; // some test case does not KylinConfig setup properly
- }
- }
-
- public static org.apache.kylin.common.util.Dictionary<?> buildDictionaryFromValueEnumerator(DictionaryInfo info, IDictionaryValueEnumerator valueEnumerator) throws IOException{
- org.apache.kylin.common.util.Dictionary dict = null;
- int baseId = 0; // always 0 for now
- final int nSamples = 5;
- ArrayList samples = Lists.newArrayListWithCapacity(nSamples);
-
- // build dict, case by data type
- DataType dataType = DataType.getInstance(info.getDataType());
- if (dataType.isDateTimeFamily())
- dict = buildDateStrDict(valueEnumerator, baseId, nSamples, samples);
- else if (dataType.isNumberFamily())
- dict = buildNumberDict(valueEnumerator, baseId, nSamples, samples);
- else
- dict = buildStringDict(valueEnumerator, baseId, nSamples, samples);
-
- // log a few samples
- StringBuilder buf = new StringBuilder();
- for (Object s : samples) {
- if (buf.length() > 0)
- buf.append(", ");
- buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
- }
- logger.info("Dictionary value samples: " + buf.toString());
- logger.info("Dictionary cardinality: " + dict.getSize());
-
- if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY)
- throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- " + info.getSourceTable() + "." + info.getSourceColumn() + " cardinality: " + dict.getSize());
-
- return dict;
- }
-
- public static org.apache.kylin.common.util.Dictionary mergeDictionaries(DictionaryInfo targetInfo, List<DictionaryInfo> sourceDicts) throws IOException {
- return buildDictionaryFromValueEnumerator(targetInfo, new MultipleDictionaryValueEnumerator(sourceDicts));
- }
-
- public static org.apache.kylin.common.util.Dictionary<?> buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException {
-
- // currently all data types are casted to string to build dictionary
- // String dataType = info.getDataType();
-
- IDictionaryValueEnumerator columnValueEnumerator = null;
- try {
- logger.info("Building dictionary " + JsonUtil.writeValueAsString(info));
-
- columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), info.getSourceColumnIndex());
- return buildDictionaryFromValueEnumerator(info, columnValueEnumerator);
- } finally {
- if (columnValueEnumerator != null)
- columnValueEnumerator.close();
- }
- }
-
- private static org.apache.kylin.common.util.Dictionary buildDateStrDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException {
- final int BAD_THRESHOLD = 2;
- String matchPattern = null;
- byte[] value;
-
- for (String ptn : DATE_PATTERNS) {
- matchPattern = ptn; // be optimistic
- int badCount = 0;
- SimpleDateFormat sdf = new SimpleDateFormat(ptn);
-
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value.length == 0)
- continue;
-
- String str = Bytes.toString(value);
- try {
- sdf.parse(str);
- if (samples.size() < nSamples && !samples.contains(str))
- samples.add(str);
- } catch (ParseException e) {
- logger.info("Unrecognized datetime value: " + str);
- badCount++;
- if (badCount > BAD_THRESHOLD) {
- matchPattern = null;
- break;
- }
- }
- }
- if (matchPattern != null) {
- return new DateStrDictionary(matchPattern, baseId);
- }
- }
- throw new IllegalStateException("Unrecognized datetime value");
- }
-
- private static org.apache.kylin.common.util.Dictionary buildStringDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException {
- TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter());
- byte[] value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null)
- continue;
- String v = Bytes.toString(value);
- builder.addValue(v);
- if (samples.size() < nSamples && !samples.contains(v))
- samples.add(v);
- }
- return builder.build(baseId);
- }
-
- private static org.apache.kylin.common.util.Dictionary buildNumberDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException {
- NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter());
- byte[] value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null)
- continue;
- String v = Bytes.toString(value);
- if (StringUtils.isBlank(v)) // empty string is null for numbers
- continue;
-
- builder.addValue(v);
- if (samples.size() < nSamples && !samples.contains(v))
- samples.add(v);
- }
- return builder.build(baseId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
deleted file mode 100644
index 645722c..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DictionaryInfo extends RootPersistentEntity {
-
- @JsonProperty("source_table")
- private String sourceTable;
- @JsonProperty("source_column")
- private String sourceColumn;
- @JsonProperty("source_column_index")
- private int sourceColumnIndex; // 0 based
- @JsonProperty("data_type")
- private String dataType;
- @JsonProperty("input")
- private TableSignature input;
- @JsonProperty("dictionary_class")
- private String dictionaryClass;
- @JsonProperty("cardinality")
- private int cardinality;
-
- transient Dictionary<?> dictionaryObject;
-
- public DictionaryInfo() {
- }
-
- public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) {
-
- this.updateRandomUuid();
-
- this.sourceTable = sourceTable;
- this.sourceColumn = sourceColumn;
- this.sourceColumnIndex = sourceColumnIndex;
- this.dataType = dataType;
- this.input = input;
- }
-
- public DictionaryInfo(DictionaryInfo other) {
-
- this.updateRandomUuid();
-
- this.sourceTable = other.sourceTable;
- this.sourceColumn = other.sourceColumn;
- this.sourceColumnIndex = other.sourceColumnIndex;
- this.dataType = other.dataType;
- this.input = other.input;
- }
-
- // ----------------------------------------------------------------------------
-
- public String getResourcePath() {
- return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn + "/" + uuid + ".dict";
- }
-
- public String getResourceDir() {
- return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn;
- }
-
- // ----------------------------------------------------------------------------
-
- // to decide if two dictionaries are built on the same table/column,
- // regardless of their signature
- public boolean isDictOnSameColumn(DictionaryInfo other) {
- return this.sourceTable.equalsIgnoreCase(other.sourceTable) && this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && this.sourceColumnIndex == other.sourceColumnIndex && this.dataType.equalsIgnoreCase(other.dataType) && this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
- }
-
- public String getSourceTable() {
- return sourceTable;
- }
-
- public void setSourceTable(String sourceTable) {
- this.sourceTable = sourceTable;
- }
-
- public String getSourceColumn() {
- return sourceColumn;
- }
-
- public void setSourceColumn(String sourceColumn) {
- this.sourceColumn = sourceColumn;
- }
-
- public int getSourceColumnIndex() {
- return sourceColumnIndex;
- }
-
- public void setSourceColumnIndex(int sourceColumnIndex) {
- this.sourceColumnIndex = sourceColumnIndex;
- }
-
- public String getDataType() {
- return dataType;
- }
-
- public void setDataType(String dataType) {
- this.dataType = dataType;
- }
-
- public TableSignature getInput() {
- return input;
- }
-
- public void setInput(TableSignature input) {
- this.input = input;
- }
-
- public String getDictionaryClass() {
- return dictionaryClass;
- }
-
- public void setDictionaryClass(String dictionaryClass) {
- this.dictionaryClass = dictionaryClass;
- }
-
- public Dictionary<?> getDictionaryObject() {
- return dictionaryObject;
- }
-
- public void setDictionaryObject(Dictionary<?> dictionaryObject) {
- this.dictionaryObject = dictionaryObject;
- }
-
- public int getCardinality() {
- return cardinality;
- }
-
- public void setCardinality(int cardinality) {
- this.cardinality = cardinality;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
deleted file mode 100644
index 6381643..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.JsonUtil;
-
-/**
- * @author yangli9
- *
- */
-public class DictionaryInfoSerializer implements Serializer<DictionaryInfo> {
-
- public static final DictionaryInfoSerializer FULL_SERIALIZER = new DictionaryInfoSerializer(false);
- public static final DictionaryInfoSerializer INFO_SERIALIZER = new DictionaryInfoSerializer(true);
-
- private boolean infoOnly;
-
- public DictionaryInfoSerializer() {
- this(false);
- }
-
- public DictionaryInfoSerializer(boolean infoOnly) {
- this.infoOnly = infoOnly;
- }
-
- @Override
- public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException {
- String json = JsonUtil.writeValueAsIndentString(obj);
- out.writeUTF(json);
-
- if (infoOnly == false)
- obj.getDictionaryObject().write(out);
- }
-
- @Override
- public DictionaryInfo deserialize(DataInputStream in) throws IOException {
- String json = in.readUTF();
- DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class);
-
- if (infoOnly == false) {
- Dictionary<?> dict;
- try {
- dict = (Dictionary<?>) ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- dict.readFields(in);
- obj.setDictionaryObject(dict);
- }
- return obj;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
deleted file mode 100644
index 9294e03..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.compress.utils.IOUtils;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.ReadableTable;
-import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-public class DictionaryManager {
-
- private static final Logger logger = LoggerFactory.getLogger(DictionaryManager.class);
-
- private static final DictionaryInfo NONE_INDICATOR = new DictionaryInfo();
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, DictionaryManager> CACHE = new ConcurrentHashMap<KylinConfig, DictionaryManager>();
-
- public static DictionaryManager getInstance(KylinConfig config) {
- DictionaryManager r = CACHE.get(config);
- if (r == null) {
- r = new DictionaryManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- }
- return r;
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- private LoadingCache<String, DictionaryInfo> dictCache;
-
- // path ==>
- // DictionaryInfo
-
- private DictionaryManager(KylinConfig config) {
- this.config = config;
- this.dictCache = CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
- .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, DictionaryInfo>() {
-
- @Override
- public DictionaryInfo load(String key) throws Exception {
- DictionaryInfo dictInfo = DictionaryManager.this.load(key, true);
- if (dictInfo == null) {
- return NONE_INDICATOR;
- } else {
- return dictInfo;
- }
- }
- });
- }
-
- public Dictionary<?> getDictionary(String resourcePath) throws IOException {
- DictionaryInfo dictInfo = getDictionaryInfo(resourcePath);
- return dictInfo == null ? null : dictInfo.getDictionaryObject();
- }
-
- public DictionaryInfo getDictionaryInfo(String resourcePath) throws IOException {
- try {
- DictionaryInfo result = dictCache.get(resourcePath);
- if (result == NONE_INDICATOR) {
- return null;
- } else {
- return result;
- }
- } catch (ExecutionException e) {
- throw new RuntimeException(e.getCause());
- }
- }
-
- public DictionaryInfo trySaveNewDict(Dictionary<?> newDict, DictionaryInfo newDictInfo) throws IOException {
-
- String dupDict = checkDupByContent(newDictInfo, newDict);
- if (dupDict != null) {
- logger.info("Identical dictionary content " + newDict + ", reuse existing dictionary at " + dupDict);
- return getDictionaryInfo(dupDict);
- }
-
- newDictInfo.setCardinality(newDict.getSize());
- newDictInfo.setDictionaryObject(newDict);
- newDictInfo.setDictionaryClass(newDict.getClass().getName());
-
- save(newDictInfo);
- dictCache.put(newDictInfo.getResourcePath(), newDictInfo);
-
- return newDictInfo;
- }
-
- public DictionaryInfo mergeDictionary(List<DictionaryInfo> dicts) throws IOException {
- if (dicts.size() == 0)
- return null;
-
- if (dicts.size() == 1)
- return dicts.get(0);
-
- DictionaryInfo firstDictInfo = null;
- int totalSize = 0;
- for (DictionaryInfo info : dicts) {
- // check
- if (firstDictInfo == null) {
- firstDictInfo = info;
- } else {
- if (!firstDictInfo.isDictOnSameColumn(info)) {
- // don't throw exception, just output warning as legacy cube segment may build dict on PK
- logger.warn("Merging dictionaries are not structurally equal : " + firstDictInfo.getResourcePath() + " and " + info.getResourcePath());
- }
- }
- totalSize += info.getInput().getSize();
- }
-
- if (firstDictInfo == null) {
- throw new IllegalArgumentException("DictionaryManager.mergeDictionary input cannot be null");
- }
-
- DictionaryInfo newDictInfo = new DictionaryInfo(firstDictInfo);
- TableSignature signature = newDictInfo.getInput();
- signature.setSize(totalSize);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("merged_with_no_original_path");
-
- String dupDict = checkDupByInfo(newDictInfo);
- if (dupDict != null) {
- logger.info("Identical dictionary input " + newDictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
- return getDictionaryInfo(dupDict);
- }
-
- // check for cases where merging dicts are actually same
- boolean identicalSourceDicts = true;
- for (int i = 1; i < dicts.size(); ++i) {
- if (!dicts.get(0).getDictionaryObject().equals(dicts.get(i).getDictionaryObject())) {
- identicalSourceDicts = false;
- break;
- }
- }
- if (identicalSourceDicts) {
- logger.info("Use one of the merging dictionaries directly");
- return dicts.get(0);
- }
-
- Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(newDictInfo, dicts);
- return trySaveNewDict(newDict, newDictInfo);
- }
-
- public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, String factColumnsPath) throws IOException {
-
- logger.info("building dictionary for " + col);
-
- Object[] tmp = decideSourceData(model, dict, col, factColumnsPath);
- String srcTable = (String) tmp[0];
- String srcCol = (String) tmp[1];
- int srcColIdx = (Integer) tmp[2];
- ReadableTable inpTable = (ReadableTable) tmp[3];
-
- if (!inpTable.exists())
- return null;
-
- DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature());
-
- String dupDict = checkDupByInfo(dictInfo);
- if (dupDict != null) {
- logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
- return getDictionaryInfo(dupDict);
- }
-
- Dictionary<?> dictionary = DictionaryGenerator.buildDictionary(dictInfo, inpTable);
-
- return trySaveNewDict(dictionary, dictInfo);
- }
-
- /**
- * Get column origin
- *
- * @return 1. source table name
- * 2. column name
- * 3. column cardinal in source table
- * 4. ReadableTable object
- */
-
- public Object[] decideSourceData(DataModelDesc model, String dict, TblColRef col, String factColumnsPath) throws IOException {
- String srcTable;
- String srcCol;
- int srcColIdx;
- ReadableTable table;
- MetadataManager metaMgr = MetadataManager.getInstance(config);
-
- // case of full table (dict on fact table)
- if (model == null) {
- srcTable = col.getTable();
- srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
- int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount();
- table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns);
- return new Object[] { srcTable, srcCol, srcColIdx, table };
- }
-
- // Decide source data of dictionary:
- // 1. If 'useDict' specifies pre-defined data set, use that
- // 2. Otherwise find a lookup table to scan through
-
- // Note FK on fact table is supported by scan the related PK on lookup
- // table
-
- //String useDict = cube.getRowkey().getDictionary(col);
-
- // normal case, source from lookup table
- if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) {
- // FK on fact table and join type is inner, use PK from lookup instead
- if (model.isFactTable(col.getTable())) {
- TblColRef pkCol = model.findPKByFK(col, "inner");
- if (pkCol != null)
- col = pkCol; // scan the counterparty PK on lookup table instead
- }
- srcTable = col.getTable();
- srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
- if (model.isFactTable(col.getTable())) {
- table = new FileTable(factColumnsPath + "/" + col.getName(), -1);
- } else {
- table = new HiveTable(metaMgr, col.getTable());
- }
- }
- // otherwise could refer to a data set, e.g. common_indicators.txt
- // (LEGACY PATH, since distinct values are collected from fact table)
- else {
- String dictDataSetPath = unpackDataSet(this.config.getTempHDFSDir(), dict);
- if (dictDataSetPath == null)
- throw new IllegalArgumentException("Unknown dictionary data set '" + dict + "', referred from " + col);
- srcTable = "PREDEFINED";
- srcCol = dict;
- srcColIdx = 0;
- table = new FileTable(dictDataSetPath, -1);
- }
-
- return new Object[] { srcTable, srcCol, srcColIdx, table };
- }
-
- private String unpackDataSet(String tempHDFSDir, String dataSetName) throws IOException {
-
- InputStream in = this.getClass().getResourceAsStream("/org/apache/kylin/dict/" + dataSetName + ".txt");
- if (in == null) // data set resource not found
- return null;
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- IOUtils.copy(in, buf);
- in.close();
- byte[] bytes = buf.toByteArray();
-
- Path tmpDataSetPath = new Path(tempHDFSDir + "/dict/temp_dataset/" + dataSetName + "_" + bytes.length + ".txt");
-
- FileSystem fs = HadoopUtil.getFileSystem(tempHDFSDir);
- boolean writtenNewFile = false;
- if (fs.exists(tmpDataSetPath) == false || fs.getFileStatus(tmpDataSetPath).getLen() != bytes.length) {
- fs.mkdirs(tmpDataSetPath.getParent());
- FSDataOutputStream out = fs.create(tmpDataSetPath);
- IOUtils.copy(new ByteArrayInputStream(bytes), out);
- out.close();
- writtenNewFile = true;
- }
-
- String qualifiedPath = tmpDataSetPath.makeQualified(fs.getUri(), new Path("/")).toString();
- if (writtenNewFile)
- logger.info("Dictionary temp data set file written to " + qualifiedPath);
- return qualifiedPath;
- }
-
- private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
- if (existings == null)
- return null;
-
- TableSignature input = dictInfo.getInput();
- for (String existing : existings) {
- DictionaryInfo existingInfo = load(existing, false); // skip cache, direct load from store
- if (input.equals(existingInfo.getInput()))
- return existing;
- }
-
- return null;
- }
-
- private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
- if (existings == null)
- return null;
-
- for (String existing : existings) {
- logger.info("Checking dup dict :" + existing);
- DictionaryInfo existingInfo = load(existing, true); // skip cache, direct load from store
- if (existingInfo == null)
- logger.info("existingInfo is null");
-
- if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject()))
- return existing;
- }
-
- return null;
- }
-
- public void removeDictionary(String resourcePath) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
- store.deleteResource(resourcePath);
- dictCache.invalidate(resourcePath);
- }
-
- public void removeDictionaries(String srcTable, String srcCol) throws IOException {
- DictionaryInfo info = new DictionaryInfo();
- info.setSourceTable(srcTable);
- info.setSourceColumn(srcCol);
-
- ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(info.getResourceDir());
- if (existings == null)
- return;
-
- for (String existing : existings)
- removeDictionary(existing);
- }
-
- void save(DictionaryInfo dict) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
- String path = dict.getResourcePath();
- logger.info("Saving dictionary at " + path);
- store.putResource(path, dict, DictionaryInfoSerializer.FULL_SERIALIZER);
- }
-
- DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
-
- logger.debug("Going to load DictionaryInfo from " + resourcePath);
- DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER);
-
- if (loadDictObj)
- logger.debug("Loaded dictionary at " + resourcePath);
-
- return info;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
deleted file mode 100644
index ecf980a..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-
-/**
- * Created by dongli on 10/28/15.
- */
-public interface IDictionaryValueEnumerator {
- byte[] current() throws IOException;
-
- boolean moveNext() throws IOException;
-
- void close() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/ISegment.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/ISegment.java b/dictionary/src/main/java/org/apache/kylin/dict/ISegment.java
deleted file mode 100644
index 68368b5..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/ISegment.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/17/14.
- */
-public interface ISegment {
-
- public abstract int getColumnLength(TblColRef col);
-
- public abstract Dictionary<String> getDictionary(TblColRef col);
-
- public String getName();
-
- public String getUuid();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
deleted file mode 100644
index 9ae08a8..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.dict.IDictionaryValueEnumerator;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 10/28/15.
- */
-public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator {
- ListIterator<byte[]> listIterator;
-
- public ListDictionaryValueEnumerator(List<byte[]> list) {
- listIterator = list.listIterator();
- }
-
- @Override
- public byte[] current() throws IOException {
- return listIterator.next();
- }
-
- @Override
- public boolean moveNext() throws IOException {
- return listIterator.hasNext();
- }
-
- @Override
- public void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
deleted file mode 100644
index 43d62a3..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Created by dongli on 10/28/15.
- */
-@SuppressWarnings("rawtypes")
-public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumerator {
- private int curDictIndex = 0;
- private Dictionary curDict;
- private int curKey;
- private byte[] curValue = null;
- private List<Dictionary> dictionaryList;
-
- public MultipleDictionaryValueEnumerator(List<DictionaryInfo> dictionaryInfoList) {
- dictionaryList = Lists.newArrayListWithCapacity(dictionaryInfoList.size());
- for (DictionaryInfo dictInfo : dictionaryInfoList) {
- dictionaryList.add(dictInfo.getDictionaryObject());
- }
- if (!dictionaryList.isEmpty()) {
- curDict = dictionaryList.get(0);
- curKey = curDict.getMinId();
- }
- }
-
- @Override
- public byte[] current() throws IOException {
- return curValue;
- }
-
- @Override
- public boolean moveNext() throws IOException {
- if (curDictIndex < dictionaryList.size() && curKey <= curDict.getMaxId()) {
- byte[] buffer = new byte[curDict.getSizeOfValue()];
- int size = curDict.getValueBytesFromId(curKey, buffer, 0);
- curValue = Bytes.copy(buffer, 0, size);
-
- if (++curKey > curDict.getMaxId()) {
- if (++curDictIndex < dictionaryList.size()) {
- curDict = dictionaryList.get(curDictIndex);
- curKey = curDict.getMinId();
- }
- }
-
- return true;
- }
- curValue = null;
- return false;
- }
-
- @Override
- public void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
deleted file mode 100644
index edab9d6..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * @author yangli9
- *
- */
-public class NumberDictionary<T> extends TrieDictionary<T> {
-
- public static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = 16;
-
- // encode a number into an order preserving byte sequence
- // for positives -- padding '0'
- // for negatives -- '-' sign, padding '9', invert digits, and terminate by ';'
- static class NumberBytesCodec {
-
- byte[] buf = new byte[MAX_DIGITS_BEFORE_DECIMAL_POINT * 2];
- int bufOffset = 0;
- int bufLen = 0;
-
- void encodeNumber(byte[] value, int offset, int len) {
- if (len == 0) {
- bufOffset = 0;
- bufLen = 0;
- return;
- }
-
- if (len > buf.length) {
- throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes");
- }
-
- boolean negative = value[offset] == '-';
-
- // terminate negative ';'
- int start = buf.length - len;
- int end = buf.length;
- if (negative) {
- start--;
- end--;
- buf[end] = ';';
- }
-
- // copy & find decimal point
- int decimalPoint = end;
- for (int i = start, j = offset; i < end; i++, j++) {
- buf[i] = value[j];
- if (buf[i] == '.' && i < decimalPoint) {
- decimalPoint = i;
- }
- }
- // remove '-' sign
- if (negative) {
- start++;
- }
-
- // prepend '0'
- int nZeroPadding = MAX_DIGITS_BEFORE_DECIMAL_POINT - (decimalPoint - start);
- if (nZeroPadding < 0 || nZeroPadding + 1 > start)
- throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect " + MAX_DIGITS_BEFORE_DECIMAL_POINT + " digits before decimal point at max.");
- for (int i = 0; i < nZeroPadding; i++) {
- buf[--start] = '0';
- }
-
- // consider negative
- if (negative) {
- buf[--start] = '-';
- for (int i = start + 1; i < buf.length; i++) {
- int c = buf[i];
- if (c >= '0' && c <= '9') {
- buf[i] = (byte) ('9' - (c - '0'));
- }
- }
- } else {
- buf[--start] = '0';
- }
-
- bufOffset = start;
- bufLen = buf.length - start;
- }
-
- int decodeNumber(byte[] returnValue, int offset) {
- if (bufLen == 0) {
- return 0;
- }
-
- int in = bufOffset;
- int end = bufOffset + bufLen;
- int out = offset;
-
- // sign
- boolean negative = buf[in] == '-';
- if (negative) {
- returnValue[out++] = '-';
- in++;
- end--;
- }
-
- // remove padding
- byte padding = (byte) (negative ? '9' : '0');
- for (; in < end; in++) {
- if (buf[in] != padding)
- break;
- }
-
- // all paddings before '.', special case for '0'
- if (in == end || !(buf[in] >= '0' && buf[in] <= '9')) {
- returnValue[out++] = '0';
- }
-
- // copy the rest
- if (negative) {
- for (; in < end; in++, out++) {
- int c = buf[in];
- if (c >= '0' && c <= '9') {
- c = '9' - (c - '0');
- }
- returnValue[out] = (byte) c;
- }
- } else {
- System.arraycopy(buf, in, returnValue, out, end - in);
- out += end - in;
- }
-
- return out - offset;
- }
- }
-
- static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
-
- // ============================================================================
-
- public NumberDictionary() { // default constructor for Writable interface
- super();
- }
-
- public NumberDictionary(byte[] trieBytes) {
- super(trieBytes);
- }
-
- private NumberBytesCodec getCodec() {
- NumberBytesCodec codec = localCodec.get();
- if (codec == null) {
- codec = new NumberBytesCodec();
- localCodec.set(codec);
- }
- return codec;
- }
-
- @Override
- protected boolean isNullObjectForm(T value) {
- return value == null || value.equals("");
- }
-
- @Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- NumberBytesCodec codec = getCodec();
- codec.encodeNumber(value, offset, len);
- return super.getIdFromValueBytesImpl(codec.buf, codec.bufOffset, codec.bufLen, roundingFlag);
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- NumberBytesCodec codec = getCodec();
- codec.bufOffset = 0;
- codec.bufLen = super.getValueBytesFromIdImpl(id, codec.buf, 0);
- return codec.decodeNumber(returnValue, offset);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
deleted file mode 100644
index c35a259..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * @author yangli9
- *
- */
-public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> {
-
- NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec();
-
- public NumberDictionaryBuilder(BytesConverter<T> bytesConverter) {
- super(bytesConverter);
- }
-
- @Override
- public void addValue(byte[] value) {
- codec.encodeNumber(value, 0, value.length);
- byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen);
- super.addValue(copy);
- }
-
- public NumberDictionary<T> build(int baseId) {
- byte[] trieBytes = buildTrieBytes(baseId);
- NumberDictionary<T> r = new NumberDictionary<T>(trieBytes);
- return r;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java b/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
deleted file mode 100644
index 83e9d2d..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Bytes;
-
-public class StringBytesConverter implements BytesConverter<String> {
-
- @Override
- public byte[] convertToBytes(String v) {
- return Bytes.toBytes(v);
- }
-
- @Override
- public String convertFromBytes(byte[] b, int offset, int length) {
- return Bytes.toString(b, offset, length);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
deleted file mode 100644
index 42f5791..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.dict.lookup.ReadableTable;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Created by dongli on 10/28/15.
- */
-public class TableColumnValueEnumerator implements IDictionaryValueEnumerator {
-
- private ReadableTable.TableReader reader;
- private int colIndex;
- private byte[] colValue;
-
- public TableColumnValueEnumerator(ReadableTable.TableReader reader, int colIndex) {
- this.reader = reader;
- this.colIndex = colIndex;
- }
-
- @Override
- public boolean moveNext() throws IOException {
- if (reader.next()) {
- String colStrValue;
- String[] split = reader.getRow();
- if (split.length == 1) {
- colStrValue = split[0];
- } else {
- // normal case
- if (split.length <= colIndex) {
- throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
- }
- colStrValue = split[colIndex];
- }
-
- colValue = Bytes.toBytes(colStrValue);
- return true;
-
- } else {
- colValue = null;
- return false;
- }
- }
-
- @Override
- public void close() throws IOException {
- if (reader != null)
- reader.close();
- }
-
- @Override
- public byte[] current() {
- return colValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
deleted file mode 100644
index 2b6d77d..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.ref.SoftReference;
-import java.util.Arrays;
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A dictionary based on Trie data structure that maps enumerations of byte[] to
- * int IDs.
- *
- * With Trie the memory footprint of the mapping is kinda minimized at the cost
- * CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
- * roughly 10 times slower, so there's a cache layer overlays on top of Trie and
- * gracefully fall back to Trie using a weak reference.
- *
- * The implementation is thread-safe.
- *
- * @author yangli9
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class TrieDictionary<T> extends Dictionary<T> {
-
- public static final byte[] HEAD_MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
- public static final int HEAD_SIZE_I = HEAD_MAGIC.length;
-
- public static final int BIT_IS_LAST_CHILD = 0x80;
- public static final int BIT_IS_END_OF_VALUE = 0x40;
-
- private static final Logger logger = LoggerFactory.getLogger(TrieDictionary.class);
-
- private byte[] trieBytes;
-
- // non-persistent part
- transient private int headSize;
- @SuppressWarnings("unused")
- transient private int bodyLen;
- transient private int sizeChildOffset;
- transient private int sizeNoValuesBeneath;
- transient private int baseId;
- transient private int maxValueLength;
- transient private BytesConverter<T> bytesConvert;
-
- transient private int nValues;
- transient private int sizeOfId;
- transient private int childOffsetMask;
- transient private int firstByteOffset;
-
- transient private boolean enableCache = true;
- transient private SoftReference<HashMap> valueToIdCache;
- transient private SoftReference<Object[]> idToValueCache;
-
- public TrieDictionary() { // default constructor for Writable interface
- }
-
- public TrieDictionary(byte[] trieBytes) {
- init(trieBytes);
- }
-
- private void init(byte[] trieBytes) {
- this.trieBytes = trieBytes;
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
- throw new IllegalArgumentException("Wrong file type (magic does not match)");
-
- try {
- DataInputStream headIn = new DataInputStream( //
- new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
- this.headSize = headIn.readShort();
- this.bodyLen = headIn.readInt();
- this.sizeChildOffset = headIn.read();
- this.sizeNoValuesBeneath = headIn.read();
- this.baseId = headIn.readShort();
- this.maxValueLength = headIn.readShort();
-
- String converterName = headIn.readUTF();
- if (!converterName.isEmpty())
- this.bytesConvert = (BytesConverter<T>) ClassUtil.forName(converterName, BytesConverter.class).newInstance();
-
- this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
- this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
- this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
- this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
- } catch (Exception e) {
- if (e instanceof RuntimeException)
- throw (RuntimeException) e;
- else
- throw new RuntimeException(e);
- }
-
- if (enableCache) {
- valueToIdCache = new SoftReference<HashMap>(new HashMap());
- idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
- }
- }
-
- @Override
- public int getMinId() {
- return baseId;
- }
-
- @Override
- public int getMaxId() {
- return baseId + nValues - 1;
- }
-
- @Override
- public int getSizeOfId() {
- return sizeOfId;
- }
-
- @Override
- public int getSizeOfValue() {
- return maxValueLength;
- }
-
- @Override
- final protected int getIdFromValueImpl(T value, int roundingFlag) {
- if (enableCache && roundingFlag == 0) {
- HashMap cache = valueToIdCache.get(); // SoftReference to skip cache gracefully when short of memory
- if (cache != null) {
- Integer id = null;
- id = (Integer) cache.get(value);
- if (id != null)
- return id.intValue();
-
- byte[] valueBytes = bytesConvert.convertToBytes(value);
- id = getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
-
- cache.put(value, id);
- return id;
- }
- }
- byte[] valueBytes = bytesConvert.convertToBytes(value);
- return getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
- }
-
- @Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag);
- int id = calcIdFromSeqNo(seq);
- if (id < 0)
- throw new IllegalArgumentException("Not a valid value: '" + bytesConvert.convertFromBytes(value, offset, len) + "'");
- return id;
- }
-
- /**
- * returns a code point from [0, nValues), preserving order of value
- *
- * @param n
- * -- the offset of current node
- * @param inp
- * -- input value bytes to lookup
- * @param o
- * -- offset in the input value bytes matched so far
- * @param inpEnd
- * -- end of input
- * @param roundingFlag
- * -- =0: return -1 if not found -- <0: return closest smaller if
- * not found, might be -1 -- >0: return closest bigger if not
- * found, might be nValues
- */
- private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
- if (o == inpEnd) // special 'empty' value
- return checkFlag(headSize, BIT_IS_END_OF_VALUE) ? 0 : roundSeqNo(roundingFlag, -1, -1, 0);
-
- int seq = 0; // the sequence no under track
-
- while (true) {
- // match the current node, note [0] of node's value has been matched
- // when this node is selected by its parent
- int p = n + firstByteOffset; // start of node's value
- int end = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1); // end of node's value
- for (p++; p < end && o < inpEnd; p++, o++) { // note matching start from [1]
- if (trieBytes[p] != inp[o]) {
- int comp = BytesUtil.compareByteUnsigned(trieBytes[p], inp[o]);
- if (comp < 0) {
- seq += BytesUtil.readUnsigned(trieBytes, n + sizeChildOffset, sizeNoValuesBeneath);
- }
- return roundSeqNo(roundingFlag, seq - 1, -1, seq); // mismatch
- }
- }
-
- // node completely matched, is input all consumed?
- boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
- if (o == inpEnd) {
- return p == end && isEndOfValue ? seq : roundSeqNo(roundingFlag, seq - 1, -1, seq); // input all matched
- }
- if (isEndOfValue)
- seq++;
-
- // find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
- if (c == headSize) // has no children
- return roundSeqNo(roundingFlag, seq - 1, -1, seq); // input only partially matched
- byte inpByte = inp[o];
- int comp;
- while (true) {
- p = c + firstByteOffset;
- comp = BytesUtil.compareByteUnsigned(trieBytes[p], inpByte);
- if (comp == 0) { // continue in the matching child, reset n and
- // loop again
- n = c;
- o++;
- break;
- } else if (comp < 0) { // try next child
- seq += BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
- if (checkFlag(c, BIT_IS_LAST_CHILD))
- return roundSeqNo(roundingFlag, seq - 1, -1, seq); // no child can match the next byte of input
- c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
- } else { // children are ordered by their first value byte
- return roundSeqNo(roundingFlag, seq - 1, -1, seq); // no child can match the next byte of input
- }
- }
- }
- }
-
- private int roundSeqNo(int roundingFlag, int i, int j, int k) {
- if (roundingFlag == 0)
- return j;
- else if (roundingFlag < 0)
- return i;
- else
- return k;
- }
-
- @Override
- final protected T getValueFromIdImpl(int id) {
- if (enableCache) {
- Object[] cache = idToValueCache.get(); // SoftReference to skip cache gracefully when short of memory
- if (cache != null) {
- int seq = calcSeqNoFromId(id);
- if (seq < 0 || seq >= nValues)
- throw new IllegalArgumentException("Not a valid ID: " + id);
- if (cache[seq] != null)
- return (T) cache[seq];
-
- byte[] value = new byte[getSizeOfValue()];
- int length = getValueBytesFromId(id, value, 0);
- T result = bytesConvert.convertFromBytes(value, 0, length);
-
- cache[seq] = result;
- return result;
- }
- }
- byte[] value = new byte[getSizeOfValue()];
- int length = getValueBytesFromId(id, value, 0);
- return bytesConvert.convertFromBytes(value, 0, length);
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- if (id < baseId || id >= baseId + nValues)
- throw new IllegalArgumentException("Not a valid ID: " + id);
-
- int seq = calcSeqNoFromId(id);
-
- return lookupValueFromSeqNo(headSize, seq, returnValue, offset);
- }
-
- /**
- * returns a code point from [0, nValues), preserving order of value, or -1
- * if not found
- *
- * @param n
- * -- the offset of current node
- * @param seq
- * -- the code point under track
- * @param returnValue
- * -- where return value is written to
- */
- private int lookupValueFromSeqNo(int n, int seq, byte[] returnValue, int offset) {
- int o = offset;
- while (true) {
- // write current node value
- int p = n + firstByteOffset;
- int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
- System.arraycopy(trieBytes, p, returnValue, o, len);
- o += len;
-
- // if the value is ended
- boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
- if (isEndOfValue) {
- seq--;
- if (seq < 0)
- return o - offset;
- }
-
- // find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
- if (c == headSize) // has no children
- return -1; // no child? corrupted dictionary!
- int nValuesBeneath;
- while (true) {
- nValuesBeneath = BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
- if (seq - nValuesBeneath < 0) { // value is under this child, reset n and loop again
- n = c;
- break;
- } else { // go to next child
- seq -= nValuesBeneath;
- if (checkFlag(c, BIT_IS_LAST_CHILD))
- return -1; // no more child? corrupted dictionary!
- p = c + firstByteOffset;
- c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
- }
- }
- }
- }
-
- private boolean checkFlag(int offset, int bit) {
- return (trieBytes[offset] & bit) > 0;
- }
-
- private int calcIdFromSeqNo(int seq) {
- if (seq < 0 || seq >= nValues)
- return -1;
- else
- return baseId + seq;
- }
-
- private int calcSeqNoFromId(int id) {
- return id - baseId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.write(trieBytes);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE + Integer.SIZE];
- in.readFully(headPartial);
-
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
- throw new IllegalArgumentException("Wrong file type (magic does not match)");
-
- DataInputStream headIn = new DataInputStream( //
- new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
- int headSize = headIn.readShort();
- int bodyLen = headIn.readInt();
- headIn.close();
-
- byte[] all = new byte[headSize + bodyLen];
- System.arraycopy(headPartial, 0, all, 0, headPartial.length);
- in.readFully(all, headPartial.length, all.length - headPartial.length);
-
- init(all);
- }
-
- @Override
- public void dump(PrintStream out) {
- out.println("Total " + nValues + " values");
- for (int i = 0; i < nValues; i++) {
- int id = calcIdFromSeqNo(i);
- T value = getValueFromId(id);
- out.println(id + " (" + Integer.toHexString(id) + "): " + value);
- }
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(trieBytes);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TrieDictionary)) {
- logger.info("Equals return false because o is not TrieDictionary");
- return false;
- }
- TrieDictionary that = (TrieDictionary) o;
- return Arrays.equals(this.trieBytes, that.trieBytes);
- }
-
- public static void main(String[] args) throws Exception {
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- b.addValue("");
- b.print();
- b.addValue("part");
- b.print();
- b.addValue("part");
- b.print();
- b.addValue("par");
- b.print();
- b.addValue("partition");
- b.print();
- b.addValue("party");
- b.print();
- b.addValue("parties");
- b.print();
- b.addValue("paint");
- b.print();
- TrieDictionary<String> dict = b.build(0);
-
- dict.dump(System.out);
-
- dict.getIdFromValueBytes(new byte[10], 0, 0);
- }
-}
[15/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
deleted file mode 100644
index 6a1f836..0000000
--- a/jdbc/pom.xml
+++ /dev/null
@@ -1,97 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-jdbc</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:JDBC</name>
- <description>Kylin JDBC Driver on Calcite Avatica</description>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-avatica</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- <appendAssemblyId>false</appendAssemblyId>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
deleted file mode 100644
index a0074b1..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.DriverVersion;
-import org.apache.calcite.avatica.Meta;
-import org.apache.calcite.avatica.UnregisteredDriver;
-import org.apache.kylin.jdbc.util.Log4jConfigurer;
-
-/**
- * <p>
- * Kylin JDBC Driver based on Calcite Avatica and Kylin restful API.<br>
- * Supported versions:
- * </p>
- * <ul>
- * <li>jdbc 4.0</li>
- * <li>jdbc 4.1</li>
- * </ul>
- *
- * <p>
- * Supported Statements:
- * </p>
- * <ul>
- * <li>{@link KylinStatementImpl}</li>
- * <li>{@link KylinPrepareStatementImpl}</li>
- * </ul>
- *
- * <p>
- * Supported properties:
- * <ul>
- * <li>user: username</li>
- * <li>password: password</li>
- * <li>ssl: true/false</li>
- * </ul>
- * </p>
- *
- * <p>
- * Driver init code sample:<br>
- *
- * <pre>
- * Driver driver = (Driver) Class.forName("org.apache.kylin.kylin.jdbc.Driver").newInstance();
- * Properties info = new Properties();
- * info.put("user", "user");
- * info.put("password", "password");
- * info.put("ssl", true);
- * Connection conn = driver.connect("jdbc:kylin://{domain}/{project}", info);
- * </pre>
- *
- * </p>
- */
-public class Driver extends UnregisteredDriver {
-
- public static final String CONNECT_STRING_PREFIX = "jdbc:kylin:";
- static {
- try {
- Log4jConfigurer.initLogger();
- DriverManager.registerDriver(new Driver());
- } catch (SQLException e) {
- throw new RuntimeException("Error occurred while registering JDBC driver " + Driver.class.getName() + ": " + e.toString());
- }
- }
-
- @Override
- protected String getConnectStringPrefix() {
- return CONNECT_STRING_PREFIX;
- }
-
- @Override
- protected DriverVersion createDriverVersion() {
- return DriverVersion.load(Driver.class, "kylin-jdbc.properties", "Kylin JDBC Driver", "unknown version", "Kylin", "unknown version");
- }
-
- @Override
- protected String getFactoryClassName(JdbcVersion jdbcVersion) {
- switch (jdbcVersion) {
- case JDBC_30:
- throw new UnsupportedOperationException();
- case JDBC_40:
- return KylinJdbcFactory.Version40.class.getName();
- case JDBC_41:
- default:
- return KylinJdbcFactory.Version41.class.getName();
- }
- }
-
- @Override
- public Meta createMeta(AvaticaConnection connection) {
- return new KylinMeta((KylinConnection) connection);
- }
-
- // @Override
- // protected Handler createHandler() {
- // return new HandlerImpl() {
- // @Override
- // public void onConnectionInit(AvaticaConnection connection) throws SQLException {
- // KylinConnection conn = (KylinConnection) connection;
- // RemoteClient runner = ((KylinJdbcFactory) factory).newRemoteClient(conn);
- // try {
- // runner.connect();
- // conn.setMetaProject(runner.getMetadata(conn.getProject()));
- // logger.debug("Connection inited.");
- // } catch (ConnectionException e) {
- // logger.error(e.getLocalizedMessage(), e);
- // throw new SQLException(e.getLocalizedMessage());
- // }
- // }
- //
- // public void onConnectionClose(AvaticaConnection connection) {
- // logger.debug("Connection closed.");
- // }
- //
- // public void onStatementExecute(AvaticaStatement statement, ResultSink resultSink) {
- // logger.debug("statement executed.");
- // }
- //
- // public void onStatementClose(AvaticaStatement statement) {
- // logger.debug("statement closed.");
- // }
- // };
- // }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
deleted file mode 100644
index 5232375..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
-
-public interface IRemoteClient extends Closeable {
-
- public static class QueryResult {
- public final List<ColumnMetaData> columnMeta;
- public final Iterable<Object> iterable;
-
- public QueryResult(List<ColumnMetaData> columnMeta, Iterable<Object> iterable) {
- this.columnMeta = columnMeta;
- this.iterable = iterable;
- }
- }
-
- /**
- * Connect to Kylin restful service. IOException will be thrown if authentication failed.
- */
- public void connect() throws IOException;
-
- /**
- * Retrieve meta data of given project.
- */
- public KMetaProject retrieveMetaData(String project) throws IOException;
-
- /**
- * Execute query remotely and get back result.
- */
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
deleted file mode 100644
index f22a515..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.jdbc;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import javax.xml.bind.DatatypeConverter;
-
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.ColumnMetaData.Rep;
-import org.apache.calcite.avatica.ColumnMetaData.ScalarType;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethodBase;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.StringRequestEntity;
-import org.apache.commons.httpclient.protocol.Protocol;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
-import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog;
-import org.apache.kylin.jdbc.KylinMeta.KMetaColumn;
-import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
-import org.apache.kylin.jdbc.KylinMeta.KMetaSchema;
-import org.apache.kylin.jdbc.KylinMeta.KMetaTable;
-import org.apache.kylin.jdbc.json.PreparedQueryRequest;
-import org.apache.kylin.jdbc.json.QueryRequest;
-import org.apache.kylin.jdbc.json.SQLResponseStub;
-import org.apache.kylin.jdbc.json.StatementParameter;
-import org.apache.kylin.jdbc.json.TableMetaStub;
-import org.apache.kylin.jdbc.json.TableMetaStub.ColumnMetaStub;
-import org.apache.kylin.jdbc.util.DefaultSslProtocolSocketFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class KylinClient implements IRemoteClient {
-
- private static final Logger logger = LoggerFactory.getLogger(KylinClient.class);
-
- private final KylinConnection conn;
- private final Properties connProps;
- private final HttpClient httpClient;
- private final ObjectMapper jsonMapper;
-
- public KylinClient(KylinConnection conn) {
- this.conn = conn;
- this.connProps = conn.getConnectionProperties();
- this.httpClient = new HttpClient();
- this.jsonMapper = new ObjectMapper();
-
- // trust all certificates
- if (isSSL()) {
- Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new DefaultSslProtocolSocketFactory(), 443));
- }
- }
-
- @SuppressWarnings("rawtypes")
- public static Class convertType(int sqlType) {
- Class result = Object.class;
-
- switch (sqlType) {
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGVARCHAR:
- result = String.class;
- break;
- case Types.NUMERIC:
- case Types.DECIMAL:
- result = BigDecimal.class;
- break;
- case Types.BIT:
- result = Boolean.class;
- break;
- case Types.TINYINT:
- result = Byte.class;
- break;
- case Types.SMALLINT:
- result = Short.class;
- break;
- case Types.INTEGER:
- result = Integer.class;
- break;
- case Types.BIGINT:
- result = Long.class;
- break;
- case Types.REAL:
- case Types.FLOAT:
- case Types.DOUBLE:
- result = Double.class;
- break;
- case Types.BINARY:
- case Types.VARBINARY:
- case Types.LONGVARBINARY:
- result = Byte[].class;
- break;
- case Types.DATE:
- result = Date.class;
- break;
- case Types.TIME:
- result = Time.class;
- break;
- case Types.TIMESTAMP:
- result = Timestamp.class;
- break;
- }
-
- return result;
- }
-
- public static Object wrapObject(String value, int sqlType) {
- if (null == value) {
- return null;
- }
-
- switch (sqlType) {
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGVARCHAR:
- return value;
- case Types.NUMERIC:
- case Types.DECIMAL:
- return new BigDecimal(value);
- case Types.BIT:
- return Boolean.parseBoolean(value);
- case Types.TINYINT:
- return Byte.valueOf(value);
- case Types.SMALLINT:
- return Short.valueOf(value);
- case Types.INTEGER:
- return Integer.parseInt(value);
- case Types.BIGINT:
- return Long.parseLong(value);
- case Types.FLOAT:
- return Float.parseFloat(value);
- case Types.REAL:
- case Types.DOUBLE:
- return Double.parseDouble(value);
- case Types.BINARY:
- case Types.VARBINARY:
- case Types.LONGVARBINARY:
- return value.getBytes();
- case Types.DATE:
- return Date.valueOf(value);
- case Types.TIME:
- return Time.valueOf(value);
- case Types.TIMESTAMP:
- return Timestamp.valueOf(value);
- }
-
- return value;
- }
-
- private boolean isSSL() {
- return Boolean.parseBoolean(connProps.getProperty("ssl", "false"));
- }
-
- private String baseUrl() {
- return (isSSL() ? "https://" : "http://") + conn.getBaseUrl();
- }
-
- private void addHttpHeaders(HttpMethodBase method) {
- method.addRequestHeader("Accept", "application/json, text/plain, */*");
- method.addRequestHeader("Content-Type", "application/json");
-
- String username = connProps.getProperty("user");
- String password = connProps.getProperty("password");
- String basicAuth = DatatypeConverter.printBase64Binary((username + ":" + password).getBytes());
- method.addRequestHeader("Authorization", "Basic " + basicAuth);
- }
-
- @Override
- public void connect() throws IOException {
- PostMethod post = new PostMethod(baseUrl() + "/kylin/api/user/authentication");
- addHttpHeaders(post);
- StringRequestEntity requestEntity = new StringRequestEntity("{}", "application/json", "UTF-8");
- post.setRequestEntity(requestEntity);
-
- httpClient.executeMethod(post);
-
- if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
- throw asIOException(post);
- }
- }
-
- @Override
- public KMetaProject retrieveMetaData(String project) throws IOException {
- assert conn.getProject().equals(project);
-
- String url = baseUrl() + "/kylin/api/tables_and_columns?project=" + project;
- GetMethod get = new GetMethod(url);
- addHttpHeaders(get);
-
- httpClient.executeMethod(get);
-
- if (get.getStatusCode() != 200 && get.getStatusCode() != 201) {
- throw asIOException(get);
- }
-
- List<TableMetaStub> tableMetaStubs = jsonMapper.readValue(get.getResponseBodyAsStream(), new TypeReference<List<TableMetaStub>>() {
- });
-
- List<KMetaTable> tables = convertMetaTables(tableMetaStubs);
- List<KMetaSchema> schemas = convertMetaSchemas(tables);
- List<KMetaCatalog> catalogs = convertMetaCatalogs(schemas);
- return new KMetaProject(project, catalogs);
- }
-
- private List<KMetaCatalog> convertMetaCatalogs(List<KMetaSchema> schemas) {
- Map<String, List<KMetaSchema>> catalogMap = new LinkedHashMap<String, List<KMetaSchema>>();
- for (KMetaSchema schema : schemas) {
- List<KMetaSchema> list = catalogMap.get(schema.tableCatalog);
- if (list == null) {
- list = new ArrayList<KMetaSchema>();
- catalogMap.put(schema.tableCatalog, list);
- }
- list.add(schema);
- }
-
- List<KMetaCatalog> result = new ArrayList<KMetaCatalog>();
- for (List<KMetaSchema> catSchemas : catalogMap.values()) {
- String catalog = catSchemas.get(0).tableCatalog;
- result.add(new KMetaCatalog(catalog, catSchemas));
- }
- return result;
- }
-
- private List<KMetaSchema> convertMetaSchemas(List<KMetaTable> tables) {
- Map<String, List<KMetaTable>> schemaMap = new LinkedHashMap<String, List<KMetaTable>>();
- for (KMetaTable table : tables) {
- String key = table.tableCat + "!!" + table.tableSchem;
- List<KMetaTable> list = schemaMap.get(key);
- if (list == null) {
- list = new ArrayList<KMetaTable>();
- schemaMap.put(key, list);
- }
- list.add(table);
- }
-
- List<KMetaSchema> result = new ArrayList<KMetaSchema>();
- for (List<KMetaTable> schemaTables : schemaMap.values()) {
- String catalog = schemaTables.get(0).tableCat;
- String schema = schemaTables.get(0).tableSchem;
- result.add(new KMetaSchema(catalog, schema, schemaTables));
- }
- return result;
- }
-
- private List<KMetaTable> convertMetaTables(List<TableMetaStub> tableMetaStubs) {
- List<KMetaTable> result = new ArrayList<KMetaTable>(tableMetaStubs.size());
- for (TableMetaStub tableStub : tableMetaStubs) {
- result.add(convertMetaTable(tableStub));
- }
- return result;
- }
-
- private KMetaTable convertMetaTable(TableMetaStub tableStub) {
- List<KMetaColumn> columns = new ArrayList<KMetaColumn>(tableStub.getColumns().size());
- for (ColumnMetaStub columnStub : tableStub.getColumns()) {
- columns.add(convertMetaColumn(columnStub));
- }
- return new KMetaTable(tableStub.getTABLE_CAT(), tableStub.getTABLE_SCHEM(), tableStub.getTABLE_NAME(), tableStub.getTABLE_TYPE(), columns);
- }
-
- private KMetaColumn convertMetaColumn(ColumnMetaStub columnStub) {
- return new KMetaColumn(columnStub.getTABLE_CAT(), columnStub.getTABLE_SCHEM(), columnStub.getTABLE_NAME(), columnStub.getCOLUMN_NAME(), columnStub.getDATA_TYPE(), columnStub.getTYPE_NAME(), columnStub.getCOLUMN_SIZE(), columnStub.getDECIMAL_DIGITS(), columnStub.getNUM_PREC_RADIX(), columnStub.getNULLABLE(), columnStub.getCHAR_OCTET_LENGTH(), columnStub.getORDINAL_POSITION(), columnStub.getIS_NULLABLE());
- }
-
- @Override
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
-
- SQLResponseStub queryResp = executeKylinQuery(sql, convertParameters(params, paramValues));
- if (queryResp.getIsException())
- throw new IOException(queryResp.getExceptionMessage());
-
- List<ColumnMetaData> metas = convertColumnMeta(queryResp);
- List<Object> data = convertResultData(queryResp, metas);
-
- return new QueryResult(metas, data);
- }
-
- private List<StatementParameter> convertParameters(List<AvaticaParameter> params, List<Object> paramValues) {
- if (params == null || params.isEmpty())
- return null;
-
- assert params.size() == paramValues.size();
-
- List<StatementParameter> result = new ArrayList<StatementParameter>();
- for (Object v : paramValues) {
- result.add(new StatementParameter(v.getClass().getCanonicalName(), String.valueOf(v)));
- }
- return result;
- }
-
- private SQLResponseStub executeKylinQuery(String sql, List<StatementParameter> params) throws IOException {
- String url = baseUrl() + "/kylin/api/query";
- String project = conn.getProject();
-
- QueryRequest request = null;
- if (null != params) {
- request = new PreparedQueryRequest();
- ((PreparedQueryRequest) request).setParams(params);
- url += "/prestate"; // means prepared statement..
- } else {
- request = new QueryRequest();
- }
- request.setSql(sql);
- request.setProject(project);
-
- PostMethod post = new PostMethod(url);
- addHttpHeaders(post);
-
- String postBody = jsonMapper.writeValueAsString(request);
- logger.debug("Post body:\n " + postBody);
- StringRequestEntity requestEntity = new StringRequestEntity(postBody, "application/json", "UTF-8");
- post.setRequestEntity(requestEntity);
-
- httpClient.executeMethod(post);
-
- if (post.getStatusCode() != 200 && post.getStatusCode() != 201) {
- throw asIOException(post);
- }
-
- return jsonMapper.readValue(post.getResponseBodyAsStream(), SQLResponseStub.class);
- }
-
- private List<ColumnMetaData> convertColumnMeta(SQLResponseStub queryResp) {
- List<ColumnMetaData> metas = new ArrayList<ColumnMetaData>();
- for (int i = 0; i < queryResp.getColumnMetas().size(); i++) {
- SQLResponseStub.ColumnMetaStub scm = queryResp.getColumnMetas().get(i);
- ScalarType type = ColumnMetaData.scalar(scm.getColumnType(), scm.getColumnTypeName(), Rep.of(convertType(scm.getColumnType())));
-
- ColumnMetaData meta = new ColumnMetaData(i, scm.isAutoIncrement(), scm.isCaseSensitive(), scm.isSearchable(), scm.isCurrency(), scm.getIsNullable(), scm.isSigned(), scm.getDisplaySize(), scm.getLabel(), scm.getName(), scm.getSchemaName(), scm.getPrecision(), scm.getScale(), scm.getTableName(), scm.getSchemaName(), type, scm.isReadOnly(), scm.isWritable(), scm.isWritable(), null);
-
- metas.add(meta);
- }
-
- return metas;
- }
-
- private List<Object> convertResultData(SQLResponseStub queryResp, List<ColumnMetaData> metas) {
- List<String[]> stringResults = queryResp.getResults();
- List<Object> data = new ArrayList<Object>(stringResults.size());
- for (String[] result : stringResults) {
- Object[] row = new Object[result.length];
-
- for (int i = 0; i < result.length; i++) {
- ColumnMetaData meta = metas.get(i);
- row[i] = wrapObject(result[i], meta.type.id);
- }
-
- data.add(row);
- }
- return (List<Object>) data;
- }
-
- private IOException asIOException(HttpMethodBase method) throws IOException {
- return new IOException(method + " failed, error code " + method.getStatusCode() + " and response: " + method.getResponseBodyAsString());
- }
-
- @Override
- public void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
deleted file mode 100644
index 3ae5d29..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.io.IOException;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.ConnectionPropertiesImpl;
-import org.apache.calcite.avatica.Meta;
-import org.apache.calcite.avatica.Meta.CursorFactory;
-import org.apache.calcite.avatica.Meta.Signature;
-import org.apache.calcite.avatica.UnregisteredDriver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class KylinConnection extends AvaticaConnection {
-
- private static final Logger logger = LoggerFactory.getLogger(KylinConnection.class);
-
- private final String baseUrl;
- private final String project;
- private final IRemoteClient remoteClient;
-
- protected KylinConnection(UnregisteredDriver driver, KylinJdbcFactory factory, String url, Properties info) throws SQLException {
- super(driver, factory, url, info);
-
- String odbcUrl = url;
- odbcUrl = odbcUrl.replace(Driver.CONNECT_STRING_PREFIX + "//", "");
- String[] temps = odbcUrl.split("/");
-
- assert temps.length == 2;
-
- this.baseUrl = temps[0];
- this.project = temps[1];
-
- logger.debug("Kylin base url " + this.baseUrl + ", project name " + this.project);
-
- this.remoteClient = factory.newRemoteClient(this);
-
- try {
- this.remoteClient.connect();
- } catch (IOException e) {
- throw new SQLException(e);
- }
- }
-
- String getBaseUrl() {
- return baseUrl;
- }
-
- String getProject() {
- return project;
- }
-
- Properties getConnectionProperties() {
- return info;
- }
-
- public boolean getAutoCommit() throws SQLException {
- if (meta.connectionSync(handle, new ConnectionPropertiesImpl()).isAutoCommit() == null)
- setAutoCommit(true);
- return super.getAutoCommit();
- }
-
- public boolean isReadOnly() throws SQLException {
- if (meta.connectionSync(handle, new ConnectionPropertiesImpl()).isReadOnly() == null)
- setReadOnly(true);
- return super.isReadOnly();
- }
-
- @Override
- public AvaticaStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- return super.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- Meta.Signature sig = mockPreparedSignature(sql);
- return factory().newPreparedStatement(this, null, sig, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- // TODO add restful API to prepare SQL, get back expected ResultSetMetaData
- Signature mockPreparedSignature(String sql) {
- List<AvaticaParameter> params = new ArrayList<AvaticaParameter>();
- int startIndex = 0;
- while (sql.indexOf("?", startIndex) >= 0) {
- AvaticaParameter param = new AvaticaParameter(false, 0, 0, 0, null, null, null);
- params.add(param);
- startIndex = sql.indexOf("?", startIndex) + 1;
- }
-
- ArrayList<ColumnMetaData> columns = new ArrayList<ColumnMetaData>();
- Map<String, Object> internalParams = Collections.<String, Object> emptyMap();
-
- return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY, Meta.StatementType.SELECT);
- }
-
- private KylinJdbcFactory factory() {
- return (KylinJdbcFactory) factory;
- }
-
- public IRemoteClient getRemoteClient() {
- return remoteClient;
- }
-
- @Override
- public void close() throws SQLException {
- super.close();
- try {
- remoteClient.close();
- } catch (IOException e) {
- throw new SQLException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
deleted file mode 100644
index dbfe36d..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.Properties;
-import java.util.TimeZone;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaDatabaseMetaData;
-import org.apache.calcite.avatica.AvaticaFactory;
-import org.apache.calcite.avatica.AvaticaPreparedStatement;
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaResultSetMetaData;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.QueryState;
-import org.apache.calcite.avatica.Meta.Frame;
-import org.apache.calcite.avatica.Meta.Signature;
-import org.apache.calcite.avatica.Meta.StatementHandle;
-import org.apache.calcite.avatica.UnregisteredDriver;
-
-/**
- * Kylin JDBC factory.
- */
-public class KylinJdbcFactory implements AvaticaFactory {
-
- public static class Version40 extends KylinJdbcFactory {
- public Version40() {
- super(4, 0);
- }
- }
-
- public static class Version41 extends KylinJdbcFactory {
- public Version41() {
- super(4, 1);
- }
- }
-
- final int major;
- final int minor;
-
- /** Creates a JDBC factory with given major/minor version number. */
- protected KylinJdbcFactory(int major, int minor) {
- this.major = major;
- this.minor = minor;
- }
-
- @Override
- public int getJdbcMajorVersion() {
- return major;
- }
-
- @Override
- public int getJdbcMinorVersion() {
- return minor;
- }
-
- @Override
- public AvaticaConnection newConnection(UnregisteredDriver driver, AvaticaFactory factory, String url, Properties info) throws SQLException {
- return new KylinConnection(driver, (KylinJdbcFactory) factory, url, info);
- }
-
- @Override
- public AvaticaDatabaseMetaData newDatabaseMetaData(AvaticaConnection connection) {
- return new AvaticaDatabaseMetaData(connection) {
- };
- }
-
- @Override
- public AvaticaStatement newStatement(AvaticaConnection connection, StatementHandle h, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- return new KylinStatement((KylinConnection) connection, h, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public AvaticaPreparedStatement newPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- return new KylinPreparedStatement((KylinConnection) connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public AvaticaResultSet newResultSet(AvaticaStatement statement, QueryState state, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException {
- AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature);
- return new KylinResultSet(statement, state, signature, resultSetMetaData, timeZone, firstFrame);
- }
-
- @Override
- public ResultSetMetaData newResultSetMetaData(AvaticaStatement statement, Signature signature) throws SQLException {
- return new AvaticaResultSetMetaData(statement, null, signature);
- }
-
- public IRemoteClient newRemoteClient(KylinConnection conn) {
- return new KylinClient(conn);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
deleted file mode 100644
index 6bfd356..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.calcite.avatica.AvaticaUtils;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.MetaImpl;
-import org.apache.calcite.avatica.MissingResultsException;
-import org.apache.calcite.avatica.NoSuchStatementException;
-import org.apache.calcite.avatica.QueryState;
-import org.apache.calcite.avatica.remote.TypedValue;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- * Implementation of Avatica interface
- */
-public class KylinMeta extends MetaImpl {
-
- private KMetaProject metaProject;
-
- public KylinMeta(KylinConnection connection) {
- super(connection);
- }
-
- private KylinConnection connection() {
- return (KylinConnection) connection;
- }
-
- // insert/update/delete go this path, ignorable for Kylin
- @Override
- public StatementHandle prepare(ConnectionHandle ch, String sql, long maxRowCount) {
- StatementHandle result = super.createStatement(ch);
- result.signature = connection().mockPreparedSignature(sql);
- return result;
- }
-
- // real execution happens in KylinResultSet.execute()
- @Override
- public ExecuteResult execute(StatementHandle sh, List<TypedValue> parameterValues, long maxRowCount) throws NoSuchStatementException {
- final MetaResultSet metaResultSet = MetaResultSet.create(sh.connectionId, sh.id, false, sh.signature, null);
- return new ExecuteResult(ImmutableList.of(metaResultSet));
- }
-
- // mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute()
- @Override
- public ExecuteResult prepareAndExecute(StatementHandle sh, String sql, long maxRowCount, PrepareCallback callback) {
- try {
- synchronized (callback.getMonitor()) {
- callback.clear();
- sh.signature = connection().mockPreparedSignature(sql);
- callback.assign(sh.signature, null, -1);
- }
- callback.execute();
- final MetaResultSet metaResultSet = MetaResultSet.create(sh.connectionId, sh.id, false, sh.signature, null);
- return new ExecuteResult(ImmutableList.of(metaResultSet));
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void closeStatement(StatementHandle h) {
- // nothing to do
- }
-
- private KMetaProject getMetaProject() {
- try {
- if (metaProject == null) {
- KylinConnection conn = connection();
- metaProject = conn.getRemoteClient().retrieveMetaData(conn.getProject());
- }
- return metaProject;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public MetaResultSet getTableTypes(ConnectionHandle ch) {
- return createResultSet(metaTableTypes, MetaTableType.class, "TABLE_TYPE");
- }
-
- @Override
- public MetaResultSet getCatalogs(ConnectionHandle ch) {
- List<KMetaCatalog> catalogs = getMetaProject().catalogs;
- return createResultSet(catalogs, KMetaCatalog.class, "TABLE_CAT");
- }
-
- @Override
- public MetaResultSet getSchemas(ConnectionHandle ch, String catalog, Pat schemaPattern) {
- List<KMetaSchema> schemas = getMetaProject().getSchemas(catalog, schemaPattern);
- return createResultSet(schemas, KMetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG");
- }
-
- @Override
- public MetaResultSet getTables(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
- List<KMetaTable> tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList);
- return createResultSet(tables, KMetaTable.class, //
- "TABLE_CAT", //
- "TABLE_SCHEM", //
- "TABLE_NAME", //
- "TABLE_TYPE", //
- "REMARKS", //
- "TYPE_CAT", //
- "TYPE_SCHEM", //
- "TYPE_NAME", //
- "SELF_REFERENCING_COL_NAME", //
- "REF_GENERATION");
- }
-
- @Override
- public MetaResultSet getColumns(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
- List<KMetaColumn> columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern);
- return createResultSet(columns, KMetaColumn.class, //
- "TABLE_CAT", //
- "TABLE_SCHEM", //
- "TABLE_NAME", //
- "COLUMN_NAME", //
- "DATA_TYPE", //
- "TYPE_NAME", //
- "COLUMN_SIZE", //
- "BUFFER_LENGTH", //
- "DECIMAL_DIGITS", //
- "NUM_PREC_RADIX", //
- "NULLABLE", //
- "REMARKS", //
- "COLUMN_DEF", //
- "SQL_DATA_TYPE", //
- "SQL_DATETIME_SUB", //
- "CHAR_OCTET_LENGTH", //
- "ORDINAL_POSITION", //
- "IS_NULLABLE", //
- "SCOPE_CATALOG", //
- "SCOPE_TABLE", //
- "SOURCE_DATA_TYPE", //
- "IS_AUTOINCREMENT", //
- "IS_GENERATEDCOLUMN");
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private MetaResultSet createResultSet(List iterable, Class clazz, String... names) {
- final List<ColumnMetaData> columns = new ArrayList<ColumnMetaData>();
- final List<Field> fields = new ArrayList<Field>();
- final List<String> fieldNames = new ArrayList<String>();
- for (String name : names) {
- final int index = fields.size();
- final String fieldName = AvaticaUtils.toCamelCase(name);
- final Field field;
- try {
- field = clazz.getField(fieldName);
- } catch (NoSuchFieldException e) {
- throw new RuntimeException(e);
- }
- columns.add(columnMetaData(name, index, field.getType()));
- fields.add(field);
- fieldNames.add(fieldName);
- }
-
- CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames);
- Signature signature = new Signature(columns, "", null, Collections.<String, Object> emptyMap(), cursorFactory, StatementType.SELECT);
- StatementHandle sh = this.createStatement(connection().handle);
- Frame frame = new Frame(0, true, iterable);
-
- return MetaResultSet.create(connection().id, sh.id, true, signature, frame);
- }
-
- // ============================================================================
-
- public static interface NamedWithChildren extends Named {
- List<? extends NamedWithChildren> getChildren();
- }
-
- public static List<? extends NamedWithChildren> searchByPatterns(NamedWithChildren parent, Pat... patterns) {
- assert patterns != null && patterns.length > 0;
-
- List<? extends NamedWithChildren> children = findChildren(parent, patterns[0]);
- if (patterns.length == 1) {
- return children;
- } else {
- List<NamedWithChildren> result = new ArrayList<NamedWithChildren>();
- Pat[] subPatterns = Arrays.copyOfRange(patterns, 1, patterns.length);
- for (NamedWithChildren c : children) {
- result.addAll(searchByPatterns(c, subPatterns));
- }
- return result;
- }
- }
-
- private static List<? extends NamedWithChildren> findChildren(NamedWithChildren parent, Pat pattern) {
- if (null == pattern.s || pattern.s.equals("%")) {
- return parent.getChildren();
- }
-
- List<NamedWithChildren> result = new ArrayList<NamedWithChildren>();
- Pattern regex = likeToRegex(pattern);
-
- for (NamedWithChildren c : parent.getChildren()) {
- if (regex.matcher(c.getName()).matches()) {
- result.add(c);
- }
- }
- return result;
- }
-
- /**
- * Converts a LIKE-style pattern (where '%' represents a wild-card,
- * escaped using '\') to a Java regex.
- */
- private static Pattern likeToRegex(Pat pattern) {
- StringBuilder buf = new StringBuilder("^");
- char[] charArray = pattern.s.toCharArray();
- int slash = -2;
- for (int i = 0; i < charArray.length; i++) {
- char c = charArray[i];
- if (slash == i - 1) {
- buf.append('[').append(c).append(']');
- } else {
- switch (c) {
- case '\\':
- slash = i;
- break;
- case '%':
- buf.append(".*");
- break;
- case '[':
- buf.append("\\[");
- break;
- case ']':
- buf.append("\\]");
- break;
- default:
- buf.append('[').append(c).append(']');
- }
- }
- }
- buf.append("$");
-
- return Pattern.compile(buf.toString());
- }
-
- // ============================================================================
-
- private static final List<MetaTableType> metaTableTypes = new ArrayList<MetaTableType>();
- static {
- metaTableTypes.add(new MetaTableType("TABLE"));
- }
-
- public static class KMetaProject implements NamedWithChildren {
- public final String projectName;
- public final List<KMetaCatalog> catalogs;
-
- public KMetaProject(String projectName, List<KMetaCatalog> catalogs) {
- this.projectName = projectName;
- this.catalogs = catalogs;
- }
-
- @SuppressWarnings("unchecked")
- public List<KMetaSchema> getSchemas(String catalog, Pat schemaPattern) {
- return (List<KMetaSchema>) searchByPatterns(this, Pat.of(catalog), schemaPattern);
- }
-
- @SuppressWarnings("unchecked")
- public List<KMetaTable> getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List<String> typeList) {
- return (List<KMetaTable>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern);
- }
-
- @SuppressWarnings("unchecked")
- public List<KMetaColumn> getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) {
- return (List<KMetaColumn>) searchByPatterns(this, Pat.of(catalog), schemaPattern, tableNamePattern, columnNamePattern);
- }
-
- @Override
- public String getName() {
- return projectName;
- }
-
- @Override
- public List<? extends NamedWithChildren> getChildren() {
- return catalogs;
- }
- }
-
- public static class KMetaCatalog implements NamedWithChildren {
- public final String tableCat;
- public final List<KMetaSchema> schemas;
-
- public KMetaCatalog(String tableCatalog, List<KMetaSchema> schemas) {
- this.tableCat = tableCatalog;
- this.schemas = schemas;
- }
-
- @Override
- public String getName() {
- return tableCat;
- }
-
- @Override
- public List<? extends NamedWithChildren> getChildren() {
- return schemas;
- }
- }
-
- public static class KMetaSchema extends MetaSchema implements NamedWithChildren {
- public final List<KMetaTable> tables;
-
- public KMetaSchema(String tableCatalog, String tableSchem, List<KMetaTable> tables) {
- super(tableCatalog, tableSchem);
- this.tables = tables;
- }
-
- @Override
- public List<? extends NamedWithChildren> getChildren() {
- return tables;
- }
- }
-
- public static class KMetaTable extends MetaTable implements NamedWithChildren {
- public final List<KMetaColumn> columns;
-
- public KMetaTable(String tableCat, String tableSchem, String tableName, String tableType, List<KMetaColumn> columns) {
- super(tableCat, tableSchem, tableName, tableType);
- this.columns = columns;
- }
-
- @Override
- public List<? extends NamedWithChildren> getChildren() {
- return columns;
- }
- }
-
- public static class KMetaColumn extends MetaColumn implements NamedWithChildren {
-
- public KMetaColumn(String tableCat, String tableSchem, String tableName, String columnName, int dataType, String typeName, int columnSize, Integer decimalDigits, int numPrecRadix, int nullable, int charOctetLength, int ordinalPosition, String isNullable) {
- super(tableCat, tableSchem, tableName, columnName, dataType, typeName, columnSize, decimalDigits, numPrecRadix, nullable, charOctetLength, ordinalPosition, isNullable);
- }
-
- @Override
- public List<NamedWithChildren> getChildren() {
- return Collections.<NamedWithChildren> emptyList();
- }
- }
-
- @Override
- public Frame fetch(StatementHandle h, long offset, int fetchMaxRowCount) throws NoSuchStatementException, MissingResultsException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public boolean syncResults(StatementHandle sh, QueryState state, long offset) throws NoSuchStatementException {
- // TODO Auto-generated method stub
- return false;
- }
-
- @Override
- public void commit(ConnectionHandle ch) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void rollback(ConnectionHandle ch) {
- // TODO Auto-generated method stub
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
deleted file mode 100644
index 098c3c2..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.jdbc;
-
-import java.io.InputStream;
-import java.io.Reader;
-import java.sql.NClob;
-import java.sql.RowId;
-import java.sql.SQLException;
-import java.sql.SQLXML;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaPreparedStatement;
-import org.apache.calcite.avatica.Meta.Signature;
-import org.apache.calcite.avatica.Meta.StatementHandle;
-
-public class KylinPreparedStatement extends AvaticaPreparedStatement {
-
- protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability);
- if (this.handle.signature == null)
- this.handle.signature = signature;
- }
-
- protected List<Object> getParameterValues2() {
- List<Object> values = new ArrayList<>(slots.length);
- for (int i = 0; i < slots.length; i++) {
- values.add(slots[i].value);
- }
- return values;
- }
-
- // ============================================================================
-
- public void setRowId(int parameterIndex, RowId x) throws SQLException {
- getSite(parameterIndex).setRowId(x);
- }
-
- public void setNString(int parameterIndex, String value) throws SQLException {
- getSite(parameterIndex).setNString(value);
- }
-
- public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
- getSite(parameterIndex).setNCharacterStream(value, length);
- }
-
- public void setNClob(int parameterIndex, NClob value) throws SQLException {
- getSite(parameterIndex).setNClob(value);
- }
-
- public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getSite(parameterIndex).setClob(reader, length);
- }
-
- public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
- getSite(parameterIndex).setBlob(inputStream, length);
- }
-
- public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
- getSite(parameterIndex).setNClob(reader, length);
- }
-
- public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
- getSite(parameterIndex).setSQLXML(xmlObject);
- }
-
- public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getSite(parameterIndex).setAsciiStream(x, length);
- }
-
- public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
- getSite(parameterIndex).setBinaryStream(x, length);
- }
-
- public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
- getSite(parameterIndex).setCharacterStream(reader, length);
- }
-
- public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
- getSite(parameterIndex).setAsciiStream(x);
- }
-
- public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
- getSite(parameterIndex).setBinaryStream(x);
- }
-
- public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
- getSite(parameterIndex).setCharacterStream(reader);
- }
-
- public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
- getSite(parameterIndex).setNCharacterStream(value);
- }
-
- public void setClob(int parameterIndex, Reader reader) throws SQLException {
- getSite(parameterIndex).setClob(reader);
- }
-
- public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
- getSite(parameterIndex).setBlob(inputStream);
- }
-
- public void setNClob(int parameterIndex, Reader reader) throws SQLException {
- getSite(parameterIndex).setNClob(reader);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
deleted file mode 100644
index 9db53f4..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.io.IOException;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.AvaticaResultSet;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.QueryState;
-import org.apache.calcite.avatica.Meta.Frame;
-import org.apache.calcite.avatica.Meta.Signature;
-import org.apache.calcite.avatica.MetaImpl;
-import org.apache.kylin.jdbc.IRemoteClient.QueryResult;
-
-public class KylinResultSet extends AvaticaResultSet {
-
- public KylinResultSet(AvaticaStatement statement, QueryState state, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) {
- super(statement, state, signature, resultSetMetaData, timeZone, firstFrame);
- }
-
- @Override
- protected AvaticaResultSet execute() throws SQLException {
-
- // skip execution if result is already there (case of meta data lookup)
- if (this.firstFrame != null) {
- return super.execute();
- }
-
- String sql = signature.sql;
- List<AvaticaParameter> params = signature.parameters;
- List<Object> paramValues = null;
- if (params != null && params.size() > 0) {
- paramValues = ((KylinPreparedStatement) statement).getParameterValues2();
- }
-
- IRemoteClient client = ((KylinConnection) statement.connection).getRemoteClient();
- QueryResult result;
- try {
- result = client.executeQuery(sql, params, paramValues);
- } catch (IOException e) {
- throw new SQLException(e);
- }
-
- columnMetaDataList.clear();
- columnMetaDataList.addAll(result.columnMeta);
-
- cursor = MetaImpl.createCursor(signature.cursorFactory, result.iterable);
- return super.execute2(cursor, columnMetaDataList);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
deleted file mode 100644
index 90b8afa..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinStatement.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import org.apache.calcite.avatica.AvaticaConnection;
-import org.apache.calcite.avatica.AvaticaStatement;
-import org.apache.calcite.avatica.Meta.StatementHandle;
-
-public class KylinStatement extends AvaticaStatement {
-
- protected KylinStatement(AvaticaConnection connection, StatementHandle h, int resultSetType, int resultSetConcurrency, int resultSetHoldability) {
- super(connection, h, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
deleted file mode 100644
index f019319..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/PreparedQueryRequest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.json;
-
-import java.util.List;
-
-public class PreparedQueryRequest extends QueryRequest {
- private List<StatementParameter> params;
-
- public List<StatementParameter> getParams() {
- return params;
- }
-
- public void setParams(List<StatementParameter> params) {
- this.params = params;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
deleted file mode 100644
index ce8feb5..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.json;
-
-public class QueryRequest {
- private String sql;
- private String project;
- private boolean acceptPartial = false;
-
- public String getSql() {
- return sql;
- }
-
- public void setSql(String sql) {
- this.sql = sql;
- }
-
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-
- public boolean isAcceptPartial() {
- return acceptPartial;
- }
-
- public void setAcceptPartial(boolean acceptPartial) {
- this.acceptPartial = acceptPartial;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
deleted file mode 100644
index ee28438..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.json;
-
-import java.io.Serializable;
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/**
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class SQLResponseStub implements Serializable {
- private static final long serialVersionUID = 1L;
-
- // private static final Logger logger =
- // LoggerFactory.getLogger(SQLResponse.class);
-
- // the data type for each column
- private List<ColumnMetaStub> columnMetas;
-
- // the results rows, each row contains several columns
- private List<String[]> results;
-
- private String cube;
-
- // if not select query, only return affected row count
- private int affectedRowCount;
-
- // if isException, the detailed exception message
- private String exceptionMessage;
-
- private boolean isException;
-
- private long duration;
-
- private boolean isPartial = false;
-
- private long totalScanCount;
-
- private boolean hitCache = false;
-
- public SQLResponseStub() {
- }
-
- public static long getSerialversionuid() {
- return serialVersionUID;
- }
-
- public List<ColumnMetaStub> getColumnMetas() {
- return columnMetas;
- }
-
- public void setColumnMetas(List<ColumnMetaStub> columnMetas) {
- this.columnMetas = columnMetas;
- }
-
- public List<String[]> getResults() {
- return results;
- }
-
- public void setResults(List<String[]> results) {
- this.results = results;
- }
-
- public String getCube() {
- return cube;
- }
-
- public void setCube(String cube) {
- this.cube = cube;
- }
-
- public int getAffectedRowCount() {
- return affectedRowCount;
- }
-
- public void setAffectedRowCount(int affectedRowCount) {
- this.affectedRowCount = affectedRowCount;
- }
-
- public boolean getIsException() {
- return isException;
- }
-
- public void setIsException(boolean isException) {
- this.isException = isException;
- }
-
- public String getExceptionMessage() {
- return exceptionMessage;
- }
-
- public void setExceptionMessage(String exceptionMessage) {
- this.exceptionMessage = exceptionMessage;
- }
-
- public long getDuration() {
- return duration;
- }
-
- public void setDuration(long duration) {
- this.duration = duration;
- }
-
- public boolean isPartial() {
- return isPartial;
- }
-
- public void setPartial(boolean isPartial) {
- this.isPartial = isPartial;
- }
-
- public long getTotalScanCount() {
- return totalScanCount;
- }
-
- public void setTotalScanCount(long totalScanCount) {
- this.totalScanCount = totalScanCount;
- }
-
- public boolean isHitCache() {
- return hitCache;
- }
-
- public void setHitCache(boolean hitCache) {
- this.hitCache = hitCache;
- }
-
- @JsonIgnoreProperties(ignoreUnknown = true)
- public static class ColumnMetaStub {
-
- private boolean isAutoIncrement;
- private boolean isCaseSensitive;
- private boolean isSearchable;
- private boolean isCurrency;
- private int isNullable;// 0:nonull, 1:nullable, 2: nullableunknown
- private boolean isSigned;
- private int displaySize;
- private String label;// AS keyword
- private String name;
- private String schemaName;
- private String catelogName;
- private String tableName;
- private int precision;
- private int scale;
- private int columnType;// as defined in java.sql.Types
- private String columnTypeName;
- private boolean isReadOnly;
- private boolean isWritable;
- private boolean isDefinitelyWritable;
-
- public ColumnMetaStub() {
- }
-
- public boolean isAutoIncrement() {
- return isAutoIncrement;
- }
-
- public void setAutoIncrement(boolean isAutoIncrement) {
- this.isAutoIncrement = isAutoIncrement;
- }
-
- public boolean isCaseSensitive() {
- return isCaseSensitive;
- }
-
- public void setCaseSensitive(boolean isCaseSensitive) {
- this.isCaseSensitive = isCaseSensitive;
- }
-
- public boolean isSearchable() {
- return isSearchable;
- }
-
- public void setSearchable(boolean isSearchable) {
- this.isSearchable = isSearchable;
- }
-
- public boolean isCurrency() {
- return isCurrency;
- }
-
- public void setCurrency(boolean isCurrency) {
- this.isCurrency = isCurrency;
- }
-
- public int getIsNullable() {
- return isNullable;
- }
-
- public void setIsNullable(int isNullable) {
- this.isNullable = isNullable;
- }
-
- public boolean isSigned() {
- return isSigned;
- }
-
- public void setSigned(boolean isSigned) {
- this.isSigned = isSigned;
- }
-
- public int getDisplaySize() {
- return displaySize;
- }
-
- public void setDisplaySize(int displaySize) {
- this.displaySize = displaySize;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getSchemaName() {
- return schemaName;
- }
-
- public void setSchemaName(String schemaName) {
- this.schemaName = schemaName;
- }
-
- public String getCatelogName() {
- return catelogName;
- }
-
- public void setCatelogName(String catelogName) {
- this.catelogName = catelogName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public void setPrecision(int precision) {
- this.precision = precision;
- }
-
- public int getScale() {
- return scale;
- }
-
- public void setScale(int scale) {
- this.scale = scale;
- }
-
- public int getColumnType() {
- return columnType;
- }
-
- public void setColumnType(int columnType) {
- this.columnType = columnType;
- }
-
- public String getColumnTypeName() {
- return columnTypeName;
- }
-
- public void setColumnTypeName(String columnTypeName) {
- this.columnTypeName = columnTypeName;
- }
-
- public boolean isReadOnly() {
- return isReadOnly;
- }
-
- public void setReadOnly(boolean isReadOnly) {
- this.isReadOnly = isReadOnly;
- }
-
- public boolean isWritable() {
- return isWritable;
- }
-
- public void setWritable(boolean isWritable) {
- this.isWritable = isWritable;
- }
-
- public boolean isDefinitelyWritable() {
- return isDefinitelyWritable;
- }
-
- public void setDefinitelyWritable(boolean isDefinitelyWritable) {
- this.isDefinitelyWritable = isDefinitelyWritable;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
deleted file mode 100644
index 766b025..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/StatementParameter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.json;
-
-public class StatementParameter {
-
- private String className;
- private String value;
-
- public StatementParameter(String className, String value) {
- this.className = className;
- this.value = value;
- }
-
- public String getClassName() {
- return className;
- }
-
- public void setClazz(String className) {
- this.className = className;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
deleted file mode 100644
index 9970dff..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/TableMetaStub.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.json;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-public class TableMetaStub {
-
- private static final long serialVersionUID = 1L;
- private String TABLE_CAT;
- private String TABLE_SCHEM;
- private String TABLE_NAME;
- private String TABLE_TYPE;
- private String REMARKS;
- private String TYPE_CAT;
- private String TYPE_SCHEM;
- private String TYPE_NAME;
- private String SELF_REFERENCING_COL_NAME;
- private String REF_GENERATION;
- @JsonProperty("columns")
- private List<ColumnMetaStub> columns = new ArrayList<ColumnMetaStub>();
-
- public String getTABLE_CAT() {
- return TABLE_CAT;
- }
-
- public void setTABLE_CAT(String tABLE_CAT) {
- TABLE_CAT = tABLE_CAT;
- }
-
- public String getTABLE_SCHEM() {
- return TABLE_SCHEM;
- }
-
- public void setTABLE_SCHEM(String tABLE_SCHEM) {
- TABLE_SCHEM = tABLE_SCHEM;
- }
-
- public String getTABLE_NAME() {
- return TABLE_NAME;
- }
-
- public void setTABLE_NAME(String tABLE_NAME) {
- TABLE_NAME = tABLE_NAME;
- }
-
- public String getTABLE_TYPE() {
- return TABLE_TYPE;
- }
-
- public void setTABLE_TYPE(String tABLE_TYPE) {
- TABLE_TYPE = tABLE_TYPE;
- }
-
- public String getREMARKS() {
- return REMARKS;
- }
-
- public void setREMARKS(String rEMARKS) {
- REMARKS = rEMARKS;
- }
-
- public String getTYPE_CAT() {
- return TYPE_CAT;
- }
-
- public void setTYPE_CAT(String tYPE_CAT) {
- TYPE_CAT = tYPE_CAT;
- }
-
- public String getTYPE_SCHEM() {
- return TYPE_SCHEM;
- }
-
- public void setTYPE_SCHEM(String tYPE_SCHEM) {
- TYPE_SCHEM = tYPE_SCHEM;
- }
-
- public String getTYPE_NAME() {
- return TYPE_NAME;
- }
-
- public void setTYPE_NAME(String tYPE_NAME) {
- TYPE_NAME = tYPE_NAME;
- }
-
- public String getSELF_REFERENCING_COL_NAME() {
- return SELF_REFERENCING_COL_NAME;
- }
-
- public void setSELF_REFERENCING_COL_NAME(String sELF_REFERENCING_COL_NAME) {
- SELF_REFERENCING_COL_NAME = sELF_REFERENCING_COL_NAME;
- }
-
- public String getREF_GENERATION() {
- return REF_GENERATION;
- }
-
- public void setREF_GENERATION(String rEF_GENERATION) {
- REF_GENERATION = rEF_GENERATION;
- }
-
- public List<ColumnMetaStub> getColumns() {
- return columns;
- }
-
- public void setColumns(List<ColumnMetaStub> columns) {
- this.columns = columns;
- }
-
- public static long getSerialversionuid() {
- return serialVersionUID;
- }
-
- public static class ColumnMetaStub {
- private String TABLE_CAT;
- private String TABLE_SCHEM;
- private String TABLE_NAME;
- private String COLUMN_NAME;
- private int DATA_TYPE;
- private String TYPE_NAME;
- private int COLUMN_SIZE;
- private int BUFFER_LENGTH;
- private int DECIMAL_DIGITS;
- private int NUM_PREC_RADIX;
- private int NULLABLE;
- private String REMARKS;
- private String COLUMN_DEF;
- private int SQL_DATA_TYPE;
- private int SQL_DATETIME_SUB;
- private int CHAR_OCTET_LENGTH;
- private int ORDINAL_POSITION;
- private String IS_NULLABLE;
- private String SCOPE_CATLOG;
- private String SCOPE_SCHEMA;
- private String SCOPE_TABLE;
- private short SOURCE_DATA_TYPE;
- private String IS_AUTOINCREMENT;
-
- public String getTABLE_CAT() {
- return TABLE_CAT;
- }
-
- public void setTABLE_CAT(String tABLE_CAT) {
- TABLE_CAT = tABLE_CAT;
- }
-
- public String getTABLE_SCHEM() {
- return TABLE_SCHEM;
- }
-
- public void setTABLE_SCHEM(String tABLE_SCHEM) {
- TABLE_SCHEM = tABLE_SCHEM;
- }
-
- public String getTABLE_NAME() {
- return TABLE_NAME;
- }
-
- public void setTABLE_NAME(String tABLE_NAME) {
- TABLE_NAME = tABLE_NAME;
- }
-
- public String getCOLUMN_NAME() {
- return COLUMN_NAME;
- }
-
- public void setCOLUMN_NAME(String cOLUMN_NAME) {
- COLUMN_NAME = cOLUMN_NAME;
- }
-
- public int getDATA_TYPE() {
- return DATA_TYPE;
- }
-
- public void setDATA_TYPE(int dATA_TYPE) {
- DATA_TYPE = dATA_TYPE;
- }
-
- public String getTYPE_NAME() {
- return TYPE_NAME;
- }
-
- public void setTYPE_NAME(String tYPE_NAME) {
- TYPE_NAME = tYPE_NAME;
- }
-
- public int getCOLUMN_SIZE() {
- return COLUMN_SIZE;
- }
-
- public void setCOLUMN_SIZE(int cOLUMN_SIZE) {
- COLUMN_SIZE = cOLUMN_SIZE;
- }
-
- public int getBUFFER_LENGTH() {
- return BUFFER_LENGTH;
- }
-
- public void setBUFFER_LENGTH(int bUFFER_LENGTH) {
- BUFFER_LENGTH = bUFFER_LENGTH;
- }
-
- public int getDECIMAL_DIGITS() {
- return DECIMAL_DIGITS;
- }
-
- public void setDECIMAL_DIGITS(int dECIMAL_DIGITS) {
- DECIMAL_DIGITS = dECIMAL_DIGITS;
- }
-
- public int getNUM_PREC_RADIX() {
- return NUM_PREC_RADIX;
- }
-
- public void setNUM_PREC_RADIX(int nUM_PREC_RADIX) {
- NUM_PREC_RADIX = nUM_PREC_RADIX;
- }
-
- public int getNULLABLE() {
- return NULLABLE;
- }
-
- public void setNULLABLE(int nULLABLE) {
- NULLABLE = nULLABLE;
- }
-
- public String getREMARKS() {
- return REMARKS;
- }
-
- public void setREMARKS(String rEMARKS) {
- REMARKS = rEMARKS;
- }
-
- public String getCOLUMN_DEF() {
- return COLUMN_DEF;
- }
-
- public void setCOLUMN_DEF(String cOLUMN_DEF) {
- COLUMN_DEF = cOLUMN_DEF;
- }
-
- public int getSQL_DATA_TYPE() {
- return SQL_DATA_TYPE;
- }
-
- public void setSQL_DATA_TYPE(int sQL_DATA_TYPE) {
- SQL_DATA_TYPE = sQL_DATA_TYPE;
- }
-
- public int getSQL_DATETIME_SUB() {
- return SQL_DATETIME_SUB;
- }
-
- public void setSQL_DATETIME_SUB(int sQL_DATETIME_SUB) {
- SQL_DATETIME_SUB = sQL_DATETIME_SUB;
- }
-
- public int getCHAR_OCTET_LENGTH() {
- return CHAR_OCTET_LENGTH;
- }
-
- public void setCHAR_OCTET_LENGTH(int cHAR_OCTET_LENGTH) {
- CHAR_OCTET_LENGTH = cHAR_OCTET_LENGTH;
- }
-
- public int getORDINAL_POSITION() {
- return ORDINAL_POSITION;
- }
-
- public void setORDINAL_POSITION(int oRDINAL_POSITION) {
- ORDINAL_POSITION = oRDINAL_POSITION;
- }
-
- public String getIS_NULLABLE() {
- return IS_NULLABLE;
- }
-
- public void setIS_NULLABLE(String iS_NULLABLE) {
- IS_NULLABLE = iS_NULLABLE;
- }
-
- public String getSCOPE_CATLOG() {
- return SCOPE_CATLOG;
- }
-
- public void setSCOPE_CATLOG(String sCOPE_CATLOG) {
- SCOPE_CATLOG = sCOPE_CATLOG;
- }
-
- public String getSCOPE_SCHEMA() {
- return SCOPE_SCHEMA;
- }
-
- public void setSCOPE_SCHEMA(String sCOPE_SCHEMA) {
- SCOPE_SCHEMA = sCOPE_SCHEMA;
- }
-
- public String getSCOPE_TABLE() {
- return SCOPE_TABLE;
- }
-
- public void setSCOPE_TABLE(String sCOPE_TABLE) {
- SCOPE_TABLE = sCOPE_TABLE;
- }
-
- public short getSOURCE_DATA_TYPE() {
- return SOURCE_DATA_TYPE;
- }
-
- public void setSOURCE_DATA_TYPE(short sOURCE_DATA_TYPE) {
- SOURCE_DATA_TYPE = sOURCE_DATA_TYPE;
- }
-
- public String getIS_AUTOINCREMENT() {
- return IS_AUTOINCREMENT;
- }
-
- public void setIS_AUTOINCREMENT(String iS_AUTOINCREMENT) {
- this.IS_AUTOINCREMENT = iS_AUTOINCREMENT;
- }
- }
-}
[30/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic
----------------------------------------------------------------------
diff --git a/dictionary/src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic b/dictionary/src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic
deleted file mode 100644
index ecc2104..0000000
--- a/dictionary/src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic
+++ /dev/null
@@ -1,139304 +0,0 @@
-OK'd
-OK'ing
-aahed
-aahing
-aahs
-aal
-aaliis
-aals
-aardwolf's
-aargh
-aarrgh
-aarrghh
-aas
-aasvogel
-aasvogels
-aba's
-abac
-abaca's
-abacs
-abacterial
-abactinal
-abactinally
-abactor
-abactors
-abaka
-abakas
-abampere's
-aband
-abandon's
-abandonee
-abandonees
-abandoner
-abandoners
-abandonments
-abapical
-abasedly
-abasements
-abaser
-abasers
-abashless
-abashments
-abasia
-abasia's
-abasias
-abask
-abatable
-abatements
-abater
-abaters
-abatis's
-abatjour
-abatjours
-abator
-abators
-abattis
-abattis's
-abattises
-abattu
-abature
-abatures
-abaxile
-abaya
-abayas
-abbacy's
-abbas
-abbotcies
-abbotcy
-abbotship
-abbotships
-abbreviators
-abbreviatory
-abbreviature
-abbs
-abcoulomb's
-abdabs
-abdicable
-abdicant
-abdicator's
-abdomina
-abdominal's
-abdominals
-abdominoplasties
-abdominoplasty
-abducens
-abducens's
-abducent's
-abducentes
-abductores
-abear
-abearing
-abears
-abecedarian's
-abegging
-abeigh
-abele's
-abelia
-abelia's
-abelian
-abelias
-abelmosk's
-aberdevine
-aberdevines
-aberrance's
-aberrancy's
-aberrant's
-aberrants
-aberrate
-aberrated
-aberrates
-aberrating
-abessives
-abetment's
-abettal
-abettals
-abeyances
-abeyancies
-abeyancy
-abfarad's
-abhenry's
-abhenrys
-abhominable
-abhorrences
-abhorrency
-abhorrer's
-abidances
-abidden
-abiders
-abidings
-abieses
-abigails
-abiogenesis's
-abiogenetical
-abiogenetically
-abiogenic
-abiogenically
-abiogenist
-abiogenist's
-abiogenists
-abiological
-abioses
-abiotically
-abiturient
-abiturients
-abjected
-abjecting
-abjections
-abjectnesses
-abjects
-abjoint
-abjointed
-abjointing
-abjoints
-abjunction
-abjunctions
-ablactation
-ablactation's
-ablatitious
-ablatival
-ablatively
-ablator
-ablators
-ablaut's
-abled
-ablegate
-ablegates
-ableism's
-ableists
-ables
-ablet
-ablets
-ablings
-ablins
-ablow
-abluent
-abluents
-ablush
-abluted
-ablutionary
-ablutomane
-ablutomanes
-abmhos
-abnegations
-abnegator's
-abnormal's
-abnormalism
-abnormals
-abnormities
-abnormous
-abo
-aboded
-abodement
-aboding
-abohm's
-aboideau
-aboideaus
-aboideaux
-aboil
-aboiteau
-aboiteaus
-aboiteaux
-abolishable
-abolishers
-abolishment's
-abolitional
-abolitionary
-abolitionisms
-abolitions
-abolla
-abollae
-abollas
-aboma
-abomas
-abomasa's
-abomasal
-abomasi
-abomasum's
-abomasus
-abomasuses
-abominableness
-abominator's
-abondance
-abondances
-abonnement
-abonnements
-aboon
-aborally
-abord
-aborded
-abording
-abords
-abore
-aboriginalism
-aboriginalities
-aboriginality
-aborne
-aborter
-aborters
-aborticide's
-abortifacient's
-abortional
-abortiveness
-abortivenesses
-abortus
-abortus's
-abortuses
-abos
-abought
-aboulia's
-aboulic
-aboundingly
-abouts
-abovementioned
-aboves
-abracadabras
-abrachia
-abrachia's
-abrachias
-abradable
-abradant's
-abrader's
-abraid
-abraided
-abraiding
-abraids
-abram
-abranchial
-abranchiates
-abrash
-abrashes
-abrasivenesses
-abraxases
-abray
-abrazo
-abrazos
-abreaction's
-abreactive
-abricock
-abridgable
-abridgeable
-abridger's
-abrim
-abrin
-abrins
-abris
-abristle
-abroads
-abrogable
-abrogative
-abrosia
-abrosias
-abruption's
-abruptnesses
-abr�g�
-abscinded
-abscinding
-abscinds
-abscisin
-abscisins
-absciss
-abscisse
-abscisses
-abscissin
-abscissins
-abscissions
-abscondence
-abscondences
-abseilers
-abseilings
-absenteeisms
-absenter
-absenters
-absentmindednesses
-absey
-absinthes
-absinthiated
-absit
-absolutenesses
-absoluter
-absolutions
-absolutisms
-absolutive
-absolutized
-absolutizes
-absolutizing
-absolutory
-absolvent
-absolvents
-absolver's
-absolvitor
-absolvitors
-absorbabilities
-absorbability
-absorbances
-absorbancies
-absorbancy
-absorbate
-absorbate's
-absorbates
-absorbedly
-absorbedness
-absorbednesses
-absorbefacients
-absorbencies
-absorber's
-absorptance's
-absorptiometer
-absorptiometers
-absorptiveness
-absorptivity's
-abstemiousnesses
-abstentionism
-abstentionist
-abstentionists
-abstentious
-absterge
-absterged
-abstergents
-absterges
-absterging
-abstersion
-abstersions
-abstersive
-abstinences
-abstinency
-abstinent's
-abstractable
-abstractednesses
-abstracter's
-abstractest
-abstractional
-abstractionism's
-abstractionist's
-abstractive
-abstractively
-abstractor
-abstractor's
-abstractors
-abstrict
-abstricted
-abstricting
-abstrictions
-abstricts
-abstrusenesses
-abstruser
-abstrusest
-abstrusities
-abstrusity
-abstrusity's
-absurd's
-absurdisms
-absurdnesses
-absurds
-abthane
-abthanes
-abubble
-abuilding
-abulia's
-abuna
-abunas
-abundancies
-abundancy
-abune
-aburst
-abusable
-abusage
-abusages
-abusion
-abusions
-abusivenesses
-abutilons
-abutter's
-abvolt's
-abwatt's
-abye
-abyeing
-abyes
-abys
-abysm's
-abyssopelagic
-academes
-academias
-academicalism
-academicals
-academicism's
-academism's
-academist
-academists
-acajou
-acajous
-acalephan
-acalephans
-acalephas
-acalephes
-acalephs
-acanaceous
-acanth
-acantha
-acanthae
-acanthas
-acanthin
-acanthine
-acanthocephalan's
-acanthopterygian's
-acanths
-acapnia
-acapnia's
-acapnias
-acappella
-acarbose
-acarboses
-acari's
-acarian
-acariasis's
-acaricidal
-acaricide's
-acarid's
-acaridan
-acaridans
-acaridean
-acarideans
-acaridian
-acaridians
-acaridomatia
-acaridomatium
-acarine
-acarine's
-acarines
-acarodomatia
-acarodomatium
-acarologies
-acarologist
-acarologists
-acarophily
-acarophobia
-acarophobia's
-acarophobias
-acarpellous
-acarpelous
-acarus's
-acatalectics
-acatalepsy
-acataleptic
-acataleptics
-acatamathesia
-acater
-acaters
-acates
-acatour
-acatours
-acaudate
-acaulescence
-acaulescences
-acauline
-acaulose
-acaulous
-acausal
-accabl�
-accedences
-acceder
-acceders
-accelerando's
-accelerants
-acceleratingly
-acceleratory
-accelerometer's
-accend
-accension
-accensions
-accentless
-accentor's
-accentuality
-accentually
-accentuations
-acceptabilities
-acceptablenesses
-acceptancy
-acceptants
-acceptedly
-acceptee
-acceptees
-accepters
-acceptilation
-acceptilations
-acceptingly
-acceptingness
-acceptingnesses
-acceptive
-acceptivity
-acceptor's
-accessibilities
-accessiblenesses
-accessorial
-accessorily
-accessoriness
-accessorinesses
-acciaccatura's
-acciaccature
-accidence's
-accidentalism
-accidentality
-accidentalness
-accidentalnesses
-accidented
-accidia
-accidias
-accidies
-accinge
-accinged
-accinges
-accinging
-accipiters
-accipitrines
-accite
-accited
-accites
-acciting
-acclaimers
-acclimatation
-acclimations
-acclimatisations
-acclimatizations
-acclivous
-accloy
-accoast
-accoasted
-accoasting
-accoasts
-accoil
-accoils
-accoladed
-accolading
-accommodable
-accommodational
-accommodationists
-accommodativeness
-accommodativenesses
-accommodator
-accommodators
-accompanier
-accompaniers
-accompanyist's
-accomplishable
-accomplishers
-accompt
-accomptable
-accomptant
-accompted
-accompting
-accompts
-accordable
-accordances
-accordancies
-accordancy
-accordantly
-accorder
-accorders
-accordiable
-accostable
-accouchement's
-accoucheur's
-accoucheuse
-accoucheuse's
-accoucheuses
-accountabilities
-accountablenesses
-accountancies
-accountantship
-accountantship's
-accountantships
-accountings
-accourt
-accourted
-accourting
-accourts
-accoustrement
-accoustrements
-accoutreing
-accoy
-accra
-accras
-accreditable
-accreditations
-accrescence
-accrescences
-accruable
-accruement
-accruement's
-accruements
-accubation
-accubations
-accultural
-acculturational
-acculturations
-accumbencies
-accumbency
-accumulable
-accumulatively
-accumulativeness
-accumulativenesses
-accuratenesses
-accurse
-accursednesses
-accurses
-accursing
-accusable
-accusal's
-accusant
-accusants
-accusatival
-accusatively
-accusatorially
-accustomary
-accustomation
-accustomations
-accustomedness
-accustomednesses
-accustrement
-accustrements
-acedia's
-aceldama
-aceldamas
-acellular
-acequia
-acequias
-acer
-aceraceous
-acerated
-acerbities
-acerola
-acerola's
-acerolas
-acerous
-acers
-acervately
-acervation
-acervations
-acervuli
-acervuli's
-acervulus
-acervulus's
-acescence
-acescency
-acescents
-aceta
-acetabula
-acetabular
-acetabulum's
-acetal's
-acetaldehyde's
-acetamid
-acetamide's
-acetamids
-acetaminophens
-acetanilid
-acetanilid's
-acetanilide's
-acetanilids
-acetated
-acetazolamide
-acetazolamides
-acetification
-acetifications
-acetifier
-acetifiers
-acetin
-acetins
-acetones
-acetonitrile
-acetonitriles
-acetophenetidin's
-acetose
-acetoxyl
-acetoxyls
-acetums
-acetyl's
-acetylation
-acetylations
-acetylative
-acetylcholine's
-acetylcholinesterase
-acetylcholinesterases
-acetylenes
-acetylenic
-acetylic
-acetylsalicylate
-acetylsalicylates
-achaenocarps
-achage
-achages
-achalasia
-achalasias
-acharn�
-acharya
-acharyas
-achenial
-achillea
-achillea's
-achilleas
-achimenes
-achimenes's
-achinesses
-aching's
-achings
-achiote
-achiotes
-achira
-achira's
-achiral
-achiras
-achkan
-achkans
-achlorhydria's
-achlorhydric
-acholia
-acholia's
-acholias
-achondrite's
-achondritic
-achondroplasia's
-achondroplastic
-achromaticities
-achromatin's
-achromatism's
-achromatopsia
-achromats
-acicula's
-aciculae
-aciculated
-aciculums
-acidanthera
-acidantheras
-acidemia's
-acidemias
-acidfreak
-acidfreaks
-acidhead
-acidheads
-acidifiable
-acidification's
-acidifiers
-acidimeters
-acidimetric
-acidimetry's
-acidities
-acidnesses
-acidophil's
-acidophile
-acidophile's
-acidophilic
-acidoses
-acidulation
-acidulations
-acidurias
-acidy
-acierage
-acierated
-acierates
-acierating
-acieration
-aciform
-acinaceous
-acinar
-acini's
-acinic
-acinose
-acinous
-acinus's
-ackee
-ackee's
-ackees
-ackers
-acknowledgeably
-acknowledgedly
-acknowledger
-acknowledgers
-aclinic
-acmatic
-acmic
-acmite
-acmites
-acnes
-acnodes
-acock
-acockbill
-acoelomate
-acoelomates
-acoelous
-acoemeti
-acold
-acolouthic
-acolouthos
-acolouthoses
-acoluthic
-aconitic
-aconitine
-aconitum
-aconitums
-acorned
-acosmist
-acosmists
-acotyledonous
-acotyledons
-acouchi
-acouchies
-acouchis
-acouchy
-acoustic's
-acoustician's
-acoustoelectric
-acoustoelectrically
-acoustooptical
-acoustooptics
-acquaintanceships
-acquest
-acquests
-acquiescences
-acquiescingly
-acquight
-acquighted
-acquighting
-acquights
-acquirability
-acquiral
-acquirals
-acquirements
-acquirer's
-acquisitional
-acquisitivenesses
-acquisitor
-acquisitors
-acquist
-acquite
-acquited
-acquites
-acquiting
-acquitment
-acquittance's
-acquitter
-acquitters
-acrasia
-acrasias
-acrasin
-acrasins
-acrawl
-acridin
-acridines
-acridities
-acridnesses
-acriflavin
-acriflavines
-acrimonies
-acrimoniousnesses
-acritarch
-acritarchs
-acroamatic
-acroamatical
-acrobatism
-acrocentric
-acrocentrics
-acrocephalic
-acrocephalies
-acrocephalous
-acrocephaly
-acrocephaly's
-acrodont's
-acrogen's
-acrogenic
-acrogenous
-acrogenously
-acrolect
-acrolectal
-acrolects
-acrolein's
-acrolithic
-acroliths
-acromegalics
-acromegaly's
-acromia
-acromial
-acromion's
-acron
-acronal
-acronic
-acronical
-acrons
-acronycal
-acronychal
-acronychally
-acronymania
-acronymically
-acropetally
-acrophobe
-acrophobes
-acrophobias
-acrophobics
-acrophonetic
-acrophonic
-acrophony
-acrosomal
-acrosome
-acrosome's
-acrosomes
-acrospires
-acrostical
-acrostically
-acroteria
-acroterial
-acroterium
-acroteriums
-acroters
-acrotic
-acrotism
-acrotisms
-acrylamides
-acrylate
-acrylate's
-acrylates
-acrylonitrile's
-acta
-actabilities
-actability
-actin's
-actinally
-actings
-actinia's
-actiniae
-actinian
-actinian's
-actinians
-actinically
-actinism's
-actiniums
-actinobacilli
-actinobacillosis
-actinobacillus
-actinoids
-actinolite's
-actinomere
-actinomeres
-actinometer's
-actinometric
-actinometries
-actinometry
-actinometry's
-actinomorphies
-actinomorphous
-actinomorphy
-actinomyces
-actinomyces's
-actinomycetal
-actinomycete's
-actinomycetous
-actinomycin's
-actinomycosis's
-actinomycotic
-actinons
-actinopod's
-actinotherapy's
-actinouraniums
-actinozoan's
-actioned
-actioner
-actioners
-actioning
-actionist
-actionists
-activenesses
-activin
-activins
-activisms
-activize
-activized
-activizes
-activizing
-actomyosin's
-acton
-actons
-actorish
-actressy
-actual's
-actualist
-actualists
-actualit�
-actualit�s
-actuations
-acture
-acuities
-aculeated
-aculei's
-aculeus's
-acumens
-acumination
-acuminations
-acuminous
-acupoint
-acupoints
-acupressures
-acupunctures
-acushla
-acushlas
-acutances
-acutenesses
-acyclovirs
-acylate
-acylated
-acylates
-acylating
-acylation
-acylations
-acyloin
-acyloins
-acyls
-adagial
-adamance's
-adamancies
-adamantean
-adamants
-adamsites
-adaptabilities
-adaptableness
-adaptablenesses
-adaptationally
-adaptative
-adaptedness
-adaptednesses
-adaptivenesses
-adaptivities
-adarbitrium
-adaw
-adays
-addax's
-addebted
-addedly
-addeem
-addenda's
-adderstone
-adderstones
-adderwort
-adderworts
-addictedness
-addio
-addios
-additaments
-addititious
-additivities
-addleheaded
-addlement
-addoom
-addorsed
-addressabilities
-addressability
-addresser
-addressers
-addressor
-addressors
-addrest
-adducent
-adducers
-adduction's
-adductive
-adductor's
-adeem
-adeemed
-adeeming
-adeems
-adelantado
-adelantados
-ademptions
-adenectomies
-adenines
-adenitis's
-adenocarcinoma's
-adenocarcinomata
-adenocarcinomatous
-adenohypophyseal
-adenohypophyses
-adenohypophysial
-adenohypophysis
-adenohypophysis's
-adenoidectomy's
-adenoma's
-adenomata
-adenomatoid
-adenomatous
-adenoses
-adenosine's
-adenosis
-adenosis's
-adenoviral
-adenovirus's
-adenyl
-adenyls
-adepter
-adeptest
-adeptnesses
-adequacies
-adequatenesses
-adequative
-adermin's
-adespota
-adessives
-adharma
-adherences
-adherend
-adherends
-adherently
-adhesional
-adhesions
-adhesivenesses
-adhibited
-adhibiting
-adhibition
-adhibitions
-adhibits
-adiaphora
-adiaphorist
-adiaphoristic
-adiaphorists
-adiaphoron
-adiathermanous
-adiathermic
-adigranth
-adipic
-adipoceres
-adipocyte
-adipocytes
-adipose's
-adiposeness
-adiposeness's
-adiposenesses
-adiposes
-adiposis
-adiposis's
-adiposity's
-adipous
-adipsin
-adipsins
-adit's
-adi�s's
-adjacencies
-adjectively
-adjoints
-adjudgements
-adjudgments
-adjunction's
-adjunctively
-adjunctly
-adjurer
-adjurers
-adjuror
-adjurors
-adjustabilities
-adjustability
-adjustably
-adjustive
-adjustmental
-adjutage
-adjutages
-adjutancies
-adjuvancy
-adjuvant's
-adland
-admass's
-admeasurements
-admeasurer
-admeasurers
-admen's
-adminicles
-adminicular
-adminiculate
-adminiculated
-adminiculates
-adminiculating
-administrants
-administratorship
-administratrices
-administratrix
-administratrixes
-admins
-admirability's
-admirableness's
-admiralship
-admiralships
-admiralties
-admirations
-admirative
-admissibilities
-admissiblenesses
-admittable
-admittee
-admittees
-admitter
-admitters
-admonisher's
-admonitive
-admonitor
-admonitorily
-admonitors
-adnascent
-adnation
-adnations
-adnexa
-adnexa's
-adnexal
-adnominal
-adnoun
-adnoun's
-adnouns
-adobelike
-adobo
-adobo's
-adobos
-adolescently
-adonis's
-adoors
-adoptabilities
-adoptability
-adoptees
-adoptianism
-adoptianisms
-adoptianist
-adoptianists
-adoptionism
-adoptionisms
-adoptionist
-adoptionists
-adoptious
-adorabilities
-adorability
-adorability's
-adorablenesses
-adoral
-adorations
-adorners
-ados
-adoze
-adpress
-adpressed
-adpresses
-adpressing
-adrad
-adread
-adred
-adrenalectomies
-adrenalectomized
-adrenalectomy
-adrenalectomy's
-adrenalines
-adrenalize
-adrenalized
-adrenalizes
-adrenalizing
-adrenally
-adrenergic
-adrenergically
-adrenochrome
-adrenochromes
-adrenocortical
-adrenocorticosteroid
-adrenocorticosteroids
-adrenocorticotrophic
-adrenocorticotrophin
-adrenocorticotrophin's
-adrenocorticotrophins
-adrenocorticotropin
-adrenocorticotropin's
-adrenocorticotropins
-adrenolytic
-adrenolytics
-adriamycin
-adroiter
-adroitest
-adroitnesses
-adry
-adscititiously
-adscript
-adscriptions
-adscripts
-adsorbabilities
-adsorbability
-adsorbate's
-adsorber
-adsorbers
-adsuki
-adsum
-adularias
-adulations
-adulterators
-adulterines
-adulterously
-adulthoods
-adultlike
-adultly
-adultness
-adultnesses
-adumbrations
-adumbratively
-adunc
-aduncate
-aduncated
-aduncity
-aduncous
-advanceman
-advancemen
-advancers
-advantageousness
-advantageousness's
-advantageousnesses
-advect
-advected
-advecting
-advection's
-advective
-advects
-advened
-advenes
-advening
-adventitia's
-adventitial
-adventitiousness
-adventitiousnesses
-adventive
-adventively
-adventives
-adventuresomely
-adventuresomeness
-adventuresomenesses
-adventurism's
-adventuristic
-adventurousnesses
-adversariness
-adversarinesses
-adversatively
-adversatives
-adversenesses
-advertence's
-advertencies
-advertency
-advertency's
-advertently
-advertisings
-advew
-adviceful
-advisabilities
-advisableness
-advisablenesses
-advisatory
-advisedness
-advisee's
-advisements
-advisership
-advisorate
-advisorates
-advocaats
-advocacies
-advocations
-advocative
-advocators
-advocatory
-advowson's
-adward
-adwoman
-adwomen
-adynamia's
-adynamic
-adyta
-adzed
-adzing
-adzuki
-adzukis
-aechmea
-aechmeas
-aecia's
-aecial
-aecidia
-aecidial
-aecidiospore
-aecidiospores
-aecidium
-aeciospore's
-aecium's
-aediles
-aedileship
-aedileships
-aedine
-aefald
-aefauld
-aegirine
-aegirite
-aegises
-aeglogues
-aegrotats
-aeneus
-aeolipyle
-aeolipyles
-aeolotropy
-aeonic
-aepyornis
-aepyornis's
-aepyornises
-aequorin
-aequorins
-aerations
-aerenchyma
-aerenchymas
-aerenchymata
-aerenchymatous
-aeriality
-aeried
-aerier
-aeriest
-aerily
-aeroballistic
-aerobat
-aerobats
-aerobe's
-aerobia
-aerobicize
-aerobicized
-aerobicizes
-aerobicizing
-aerobiological
-aerobiologically
-aerobiologies
-aerobiologist
-aerobiologists
-aerobiont
-aerobionts
-aerobiosis's
-aerobiotic
-aerobiotically
-aerobium
-aerobomb
-aerobombs
-aerobrake
-aerobraked
-aerobrakes
-aerobus
-aerobuses
-aeroculture
-aerocultures
-aerodart
-aerodarts
-aeroduct
-aeroducts
-aerodynamicists
-aerodynes
-aeroelastic
-aeroelastician
-aeroelasticians
-aeroelasticities
-aeroelasticity
-aeroembolism's
-aerogels
-aerogenerator
-aerogenerator's
-aerogenerators
-aerogram's
-aerographers
-aerographs
-aerohydroplane
-aerohydroplanes
-aerolite's
-aerolith
-aerolithology
-aeroliths
-aerolitic
-aerologists
-aeromagnetic
-aeromagnetically
-aeromagnetics
-aeromechanical
-aeromechanically
-aeromedicine
-aeromedicine's
-aeromedicines
-aerometeorographs
-aerometries
-aeromotor
-aeromotors
-aeronaut's
-aeronautically
-aeroneuroses
-aeronomer
-aeronomers
-aeronomic
-aeronomical
-aeronomies
-aeronomist
-aeronomists
-aeronomy
-aeropauses
-aerophagia's
-aerophobe
-aerophobes
-aerophobias
-aerophobic
-aerophones
-aerophore
-aerophores
-aerophyte's
-aeroplankton
-aerosat
-aerosats
-aeroshell
-aeroshells
-aerosiderite
-aerosolization
-aerosolizations
-aerosolize
-aerosolized
-aerosolizes
-aerosolizing
-aerospaces
-aerospheres
-aerostatical
-aerostats
-aerotactic
-aerotaxis
-aerothermodynamic
-aerotrain
-aerotrains
-aerotropic
-aerotropism
-aesces
-aesir
-aesthetic's
-aestheticisms
-aestheticist
-aestheticists
-aether's
-aethereal
-aethrioscope
-aethrioscopes
-aetiological
-afar's
-afara
-afaras
-afars
-afear
-afeared
-afearing
-afears
-affabilities
-affabler
-affablest
-affaire's
-affear
-affeard
-affeare
-affeared
-affearing
-affears
-affectabilities
-affectability
-affectable
-affectedness
-affectedness's
-affectednesses
-affecter
-affecters
-affectionally
-affectionateness
-affectionateness's
-affectionatenesses
-affectioned
-affectioning
-affectionless
-affectivities
-affectivity
-affectlessly
-affectlessnesses
-affeered
-affeering
-affeerment
-affeers
-affenpinscher's
-afferent's
-afferently
-afferents
-affettuosos
-affiance's
-affiants
-affiches
-affied
-affiliable
-affinely
-affines
-affirmably
-affirmance
-affirmances
-affirmants
-affirmers
-affirmingly
-affixable
-affixal
-affixation's
-affixer
-affixers
-affixial
-affixment
-affixments
-afflated
-afflation
-afflations
-afflatuses
-afflicter
-afflicters
-afflictings
-affluences
-affluencies
-affluency
-affluent's
-affluentness
-affluents
-affluenza
-affluxes
-affluxion
-affluxions
-afforce
-afforced
-afforcement
-afforcements
-afforces
-afforcing
-affordabilities
-afforestable
-afforestations
-affranchisement
-affrap
-affrayed
-affrayer
-affrayers
-affraying
-affreightment
-affreightments
-affret
-affricate's
-affricated
-affrication
-affrication's
-affrications
-affricative's
-affrightedly
-affrighten
-affrightened
-affrightening
-affrightens
-affrightful
-affrightment
-affrightments
-affrontingly
-affrontings
-affrontive
-affront�
-affront�e
-affusion's
-affy
-afghani's
-aficionada
-aficionadas
-afikomen
-afikomens
-aflaj
-aflatoxins
-aforehand
-aforethought's
-aforethoughts
-afortiori
-afreets
-afrits
-afro
-afront
-afrormosia
-afrormosias
-afros
-aftercares
-afterclaps
-aftercrop
-afterdamp's
-afterdeck's
-afterdinner
-aftereye
-aftergame
-aftergames
-aftergrass
-aftergrasses
-aftergrowths
-afterhours
-afterings
-afterlight
-afterpains
-afterpains's
-afterpiece's
-aftersales
-aftersensation's
-aftershaft's
-aftersupper
-afterswarm
-afterswarms
-aftertax
-aftertimes
-afterworld's
-aftosa
-aftosas
-ag
-agacerie
-agalactia
-agalactia's
-agallochs
-agalmatolite
-agalwood
-agalwoods
-agama's
-agametes
-agami
-agamically
-agamid
-agamid's
-agamids
-agamies
-agamis
-agammaglobulinemia
-agammaglobulinemia's
-agammaglobulinemias
-agammaglobulinemic
-agamogenesis's
-agamoid
-agamoids
-agamospermies
-agamospermy
-agamous
-agamy
-agapae
-agapae's
-agapanthus's
-agapeic
-agapes
-agaric's
-agarose
-agaroses
-agars
-agas
-agast
-agateware's
-agathodaimon
-agatized
-agatizing
-agatoid
-agaves
-agaze
-agazed
-aga�ant
-aga�ante
-agedly
-agedness
-agedness's
-agednesses
-ageisms
-agelast
-agelastic
-agelasts
-agelessnesses
-agelong
-agemate
-agemates
-agen
-agendaless
-agendum
-agendum's
-agendums
-agene
-agenes
-agenesia
-agenesia's
-agenesias
-agenesis's
-agenetic
-agenized
-agenizes
-agenizing
-agented
-agenting
-agentings
-agentives
-agentivity
-agentries
-agentry
-agers
-aggada
-aggadah
-aggadahs
-aggadic
-aggadot
-aggers
-aggies
-aggiornamenti
-aggiornamentos
-agglomerator
-agglomerators
-agglutinabilities
-agglutinability
-agglutinable
-agglutinant
-agglutinants
-agglutinin's
-agglutinogen's
-agglutinogenic
-aggrace
-aggraces
-aggracing
-aggradation
-aggradational
-aggradations
-aggrate
-aggrated
-aggrates
-aggrating
-aggravators
-aggregately
-aggregateness
-aggregatenesses
-aggregational
-aggregatively
-aggregator
-aggregators
-aggressivenesses
-aggressivities
-aggressivity
-aggri
-aggrievedness
-aggrievednesses
-aggrievements
-aggro's
-aggros
-aggry
-aghas
-agila
-agilas
-agilenesses
-agiler
-agilest
-agilities
-agin
-aginner
-aginners
-agio's
-agiotage's
-agisted
-agister
-agisters
-agisting
-agistment
-agistments
-agistor
-agistors
-agists
-agitable
-agitas
-agitational
-agitative
-agitpop
-agitprops
-aglare
-aglee
-aglet's
-agloo
-agloos
-aglossia
-aglossias
-aglu
-aglus
-agly
-aglycon
-aglycone
-aglycones
-aglycons
-agmas
-agnail's
-agname
-agnamed
-agnames
-agnate's
-agnathan
-agnathan's
-agnatical
-agnatically
-agnation's
-agnomens
-agnomina
-agnominal
-agnosia's
-agnostically
-agnosticisms
-agoge
-agoges
-agogic
-agogics
-agoing
-agonal
-agones
-agonist's
-agonistes
-agonistical
-agonistically
-agonistics
-agonothetes
-agons
-agood
-agora's
-agoraphobes
-agoraphobias
-agorot
-agoroth
-agouta
-agoutas
-agouti's
-agouties
-agouty
-agrafe
-agrafes
-agraffes
-agranulocyte
-agranulocytes
-agranulocytosis's
-agranulosis
-agranulosis's
-agrapha's
-agraphia's
-agraphic
-agraphon
-agrarianisms
-agraste
-agravic
-agreeabilities
-agreeability
-agreeability's
-agreeablenesses
-agrestal
-agrestial
-agria
-agrias
-agribusinessmen
-agrichemical
-agrichemicals
-agricide
-agricides
-agricultures
-agrimony's
-agrin
-agriology
-agriproduct
-agriproducts
-agrobiologic
-agrobiological
-agrobiologically
-agrobiologist
-agrobiologists
-agrobiology's
-agrobusiness
-agroforester
-agroforesters
-agroforestries
-agroforestry
-agroindustrial
-agroindustries
-agroindustry
-agrologic
-agrological
-agrologist
-agrologists
-agrology's
-agronomial
-agronomies
-agrostological
-agrostologies
-agrostologist
-agrostologists
-agrypnia
-agrypnia's
-agrypnias
-agr�gation
-agr�gations
-agr�g�
-agr�g�s
-agr�mens
-agr�ment
-agr�ments
-ags
-aguacate
-aguacates
-aguardiente
-aguardientes
-agued
-aguelike
-agueproof
-agues
-agueweed's
-aguishly
-aguishness
-aguishnesses
-aguti
-agutis
-ahas
-aheap
-ahed
-aheight
-ahems
-ahh
-ahhed
-ahhing
-ahhs
-ahigh
-ahimsas
-ahind
-ahing
-ahint
-ahistoric
-ahistorical
-ahold
-aholds
-ahorse
-ahorseback
-ahoys
-ahs
-ahull
-ahungered
-ahungry
-ai's
-aia
-aias
-aiblins
-aichmophobia
-aidance
-aidances
-aidant
-aidful
-aidless
-aidman
-aidmen
-aidos
-aiglet's
-aigret's
-aiguilles
-aiguillettes
-aikido's
-aikona
-ail's
-ailanthic
-ailanthus's
-ailanto
-ailantos
-ailette
-ailettes
-ailourophile
-ailourophiles
-ailourophilia
-ailourophilic
-ailourophobe
-ailourophobes
-ailourophobia
-ailourophobic
-ailurophiles
-ailurophilia
-ailurophilic
-ailurophobes
-ailurophobia
-ailurophobia's
-ailurophobias
-ailurophobic
-aimer
-aimers
-aimful
-aimfully
-aimlessnesses
-aina
-ainas
-ains
-ainsell
-ainsells
-airboats
-airbound
-airbrake
-airbrake's
-airbrick
-airbursts
-aircheck
-airchecks
-aircoach
-aircoaches
-aircondition
-airconditioned
-airconditioning
-airconditions
-aircraftman's
-aircraftmen's
-aircraftsman
-aircraftsman's
-aircraftsmen
-aircraftswoman
-aircraftswomen
-aircraftwoman
-aircraftwomen
-aircrew's
-aircrewman's
-airdate
-airdates
-airdrawn
-airdrome's
-airer
-airers
-airest
-airflows
-airforce
-airforce's
-airframe's
-airfreighted
-airfreighting
-airfreights
-airgap
-airglows
-airgraph
-airgraphs
-airheaded
-airhole
-airholes
-airinesses
-airish
-airlessnesses
-airmanship
-airmanship's
-airmanships
-airmen's
-airn
-airned
-airning
-airns
-airpack
-airpacks
-airpark
-airparks
-airplays
-airpost
-airpost's
-airposts
-airpowers
-airproof
-airproofed
-airproofing
-airproofs
-airscape
-airscapes
-airscrew's
-airshaft
-airshafts
-airshed
-airsheds
-airshot
-airshots
-airsicknesses
-airside
-airsides
-airspaces
-airspeed's
-airstop
-airstops
-airstream's
-airstreams
-airth
-airthed
-airthing
-airths
-airtightness
-airtightnesses
-airtimes
-airward
-airwards
-airwave's
-airwise
-airwoman's
-airwomen's
-airworthier
-airworthiest
-airworthinesses
-aisled
-aisleway
-aisleways
-aisling
-aitchbone's
-aits
-aitu
-aitus
-aiver
-aivers
-aizle
-aizles
-ajee
-ajiva
-ajivas
-ajowan
-ajowans
-ajuga
-ajugas
-ajutage
-ajutages
-ajwan
-ajwans
-akaryote
-akaryote's
-akaryotes
-ake
-aked
-akedah
-akee's
-akela
-akelas
-akenes
-akes
-akineses
-akinesia
-akinesia's
-akinesias
-akinesis
-akinesis's
-akinetic
-aking
-akkas
-akolouthos
-akolouthoses
-akoluthos
-akoluthoses
-akrasia
-akrasias
-akratic
-ala's
-alaap
-alabamine
-alabandine
-alabandite
-alabasters
-alabastrine
-alablaster
-alackaday
-alacks
-alacridus
-alacrities
-alalagmoi
-alalagmos
-alalia
-alalia's
-alamedas
-alamo
-alamort
-alamos
-alan
-aland
-alands
-alane
-alang
-alangs
-alanin
-alanine's
-alanins
-alannah
-alannahs
-alans
-alant
-alants
-alanui
-alanuis
-alanyl
-alanyls
-alap
-alapa
-alarmclock
-alarmedly
-alarmism's
-alases
-alaska
-alaskas
-alastor
-alastors
-alastrim
-alated
-alates
-alation
-alations
-alay
-alayed
-alaying
-alays
-albarelli
-albarello
-albarellos
-albas
-albatas
-albe
-albedo's
-albee
-alberghi
-albergo
-albert
-albertites
-alberts
-albescence
-albescences
-albespine
-albespines
-albespyne
-albespynes
-albicore
-albicores
-albinal
-albiness
-albinic
-albinisms
-albinistic
-albinoism
-albinotic
-albite's
-albitic
-albizia
-albizia's
-albizias
-albizzia
-albizzia's
-albizzias
-albricias
-albugineous
-albugo
-albugos
-albumenise
-albumenised
-albumenises
-albumenising
-albumens
-albuminates
-albuminoid's
-albuminoidal
-albumins
-albuminuria's
-albuminuric
-albumoses
-alburnous
-alburnums
-alcade
-alcades
-alcahest's
-alcaicer�a
-alcaicer�as
-alcaides
-alcaldes
-alcarraza
-alcarrazas
-alcatras
-alcatrases
-alcayde
-alcaydes
-alchemies
-alchemistic
-alchemistical
-alchera
-alchymies
-alchymy
-alcid
-alcidine
-alcids
-alcoholicities
-alcoholisms
-alcoholometers
-alcoholometries
-alcoholometry
-alcopop
-alcopops
-alcorza
-alcorzas
-alcoved
-alcyonarian
-alcyonarians
-alc�zar's
-aldea
-aldehyde's
-aldehydic
-alderflies
-alderfly
-alderfly's
-aldermancies
-aldermancy
-aldermanic
-aldermanity
-aldermanlike
-aldermanly
-aldermanry
-aldermanship
-aldermanships
-aldermen's
-aldern
-alderwomen's
-aldicarb
-aldicarbs
-aldohexose
-aldol's
-aldolase
-aldolases
-aldolization
-aldolizations
-aldopentose
-aldose's
-aldosterone's
-aldosteronism
-aldosteronism's
-aldosteronisms
-aldrins
-alebench
-alebenches
-aleberry
-alec
-aleconner
-alecost
-alecost's
-alecosts
-alecs
-alectryon
-alectryons
-alef
-alefs
-aleft
-alegars
-alegge
-alegges
-alehoof
-alehoof's
-alembicated
-alembication
-alembications
-alembroth
-alencon
-alencons
-alength
-aleph's
-alepine
-alepole
-alerce
-alerces
-alerions
-alerter
-alertest
-alertnesses
-alestake
-alethic
-aleuron
-aleurone's
-aleuronic
-aleurons
-alevins
-alew
-alewashed
-alexander
-alexanders
-alexandrines
-alexandrite's
-alexia's
-alexic
-alexic's
-alexine
-alexines
-alexins
-alexipharmakon
-alexipharmakons
-alexipharmics
-alfa
-alfaki
-alfakis
-alfalfas
-alfaquin
-alfaquins
-alfaquis
-alfaqu�
-alfas
-alfilaria's
-alfileria
-alfileria's
-alfilerias
-alforjas
-alfredo
-alf�rez
-alf�rezes
-algae's
-algaecide
-algaecides
-algaroba
-algarobas
-algarroba's
-algarrobo
-algarrobos
-algas
-algate
-algates
-algebraist's
-algerine
-algerines
-algesis
-algicidal
-algicides
-algidities
-algidity
-algin's
-alginates
-alginic
-algolagniac
-algolagniacs
-algolagnias
-algolagnic
-algolagnist
-algolagnists
-algological
-algologically
-algologies
-algologist
-algologists
-algometer's
-algometric
-algometrical
-algometries
-algometry
-algometry's
-algophobia's
-algorism's
-algorismic
-algors
-alguacil
-alguacils
-alguazil
-alguazils
-algum
-algums
-aliasings
-alibies
-alicant
-alicants
-alidad
-alidades
-alidads
-alienabilities
-alienage's
-alienations
-alienators
-alienees
-aliener
-alieners
-alienism's
-alienly
-aliennesses
-alienors
-alif
-alifs
-alightment
-alightments
-alikeness's
-alimental
-alimentally
-alimentation's
-alimentative
-alimentiveness
-alimonies
-alineation
-alineations
-aliner
-aliners
-alipeds
-aliquant's
-aliquot's
-alisma
-alismaceous
-alismas
-alist
-aliteracies
-aliteracy
-aliterate
-aliterate's
-aliterates
-alivenesses
-aliyos
-aliyot
-aliyoth
-alizari
-alizarin's
-alizarine
-alizarine's
-alizarines
-alizaris
-alkahest's
-alkahestic
-alkahestical
-alkalescence
-alkalescences
-alkalescencies
-alkalescency
-alkalescent
-alkalic
-alkalimeters
-alkalimetric
-alkalimetrically
-alkalimetries
-alkalimetry
-alkalimetry's
-alkalin
-alkalinities
-alkalosis's
-alkalotic
-alkane's
-alkanet's
-alkene's
-alkie
-alkies
-alkine
-alkines
-alkoxide
-alkoxides
-alkoxy
-alkyl's
-alkylate
-alkylated
-alkylates
-alkylating
-alkylations
-alkylic
-alkynes
-allamanda
-allamanda's
-allamandas
-allanites
-allantoic
-allantoidal
-allantoides
-allantoids
-allantoin
-allantoins
-allantois's
-allatives
-allayer
-allayers
-allayings
-allayment
-allayments
-allcomers
-allegers
-allegge
-allegges
-allegiant
-allegiants
-allegoricalness
-allegoricalnesses
-allel
-allelism
-allelisms
-allelomorph's
-allelomorphic
-allelomorphism
-allelomorphisms
-allelopathies
-allels
-alleluiah
-alleluiahs
-allemande's
-allenarly
-allergenicities
-allergin
-allergins
-allerion
-allerions
-allethrins
-alleviators
-alleviatory
-alleycat
-alleycats
-alleyed
-allgood
-allgood's
-allheals
-alliable
-allice
-allice's
-allices
-allicin
-allicins
-alligate
-alligated
-alligates
-alligating
-alligation
-alligations
-alligatored
-alligatoring
-allineation
-allineations
-allis
-allis's
-allises
-alliterativenesses
-alliums
-allnight
-allnighter
-alloantibodies
-alloantibody
-alloantigen
-alloantigens
-allobar
-allobars
-allocarpy
-allocheiria
-allochiria
-allocutions
-allod
-allodia
-allodiums
-allods
-allogamous
-allogamy's
-allogeneic
-allogeneically
-allogenic
-allograft
-allograft's
-allografted
-allografting
-allografts
-allograph's
-allographic
-alloiostrophos
-allomerism's
-allomerous
-allometric
-allometry's
-allomone
-allomones
-allomorph's
-allomorphisms
-allonges
-allons
-allonymous
-allonymously
-allonyms
-allopathically
-allopathists
-allopaths
-allopathy's
-allopatrically
-allopatries
-allopatry
-allophanes
-allophone's
-allophonically
-alloplasms
-alloplastic
-allopolyploid
-allopolyploidies
-allopolyploids
-allopolyploidy
-allopurinol
-allopurinol's
-allopurinols
-allosaur
-allosaur's
-allosaurs
-allosaurus
-allosaurus's
-allosauruses
-allosteric
-allosterically
-allosteries
-allostery
-allotetraploid
-allotetraploidies
-allotetraploids
-allotetraploidy
-allotheism
-allotransplant
-allotransplanted
-allotransplanting
-allotransplants
-allotriomorphic
-allotrope's
-allotropically
-allotropism
-allotropism's
-allotropisms
-allotropous
-allotropy's
-allottees
-allotters
-allotype
-allotypes
-allotypic
-allotypically
-allotypies
-allotypy
-allover's
-allovers
-allowability
-allowableness
-allowanced
-allowancing
-alloxan
-alloxans
-allround
-alls
-allseeds
-allsorts
-allspices
-allthing
-allurer
-allurers
-allusivenesses
-alluvials
-alluvion's
-allwork
-allyl's
-allylic
-all�es
-alma
-almacantar
-almacantars
-almagest
-almagests
-almah
-almahs
-almain
-almandine's
-almandite's
-almas
-alme
-almeh
-almehs
-almemars
-almeries
-almery
-almes
-almightinesses
-almirah
-almirahs
-almner
-almners
-almondy
-almonries
-almous
-almsdeed
-almsfolk
-almsgiver's
-almsgiving
-almsgivings
-almsmen
-almswomen
-almucantars
-almuces
-almud
-almude
-almudes
-almuds
-almug
-almugs
-alnage
-alnager
-alnagers
-alnages
-alnico
-alnico's
-alnicoes
-alnicos
-alocasia
-alocasia's
-alocasias
-alod
-alodia
-alodial
-alodiums
-alods
-aloed
-aloeswoods
-aloetic
-aloetics
-alogia
-alogical
-alogically
-alogicalness
-alogicalnesses
-aloins
-aloneness's
-alongshoreman
-alongshoremen
-alongst
-aloofnesses
-alopecia's
-alopecic
-alopecoid
-alowe
-alpargata
-alpargatas
-alpeen
-alpeens
-alpenglows
-alpenhorns
-alpenstock's
-alphabetarian
-alphabetarians
-alphabeted
-alphabetiform
-alphabeting
-alphamerical
-alphamerically
-alphametic
-alphametics
-alphanumerics
-alphasort
-alphasorted
-alphasorting
-alphasorts
-alphorns
-alphoses
-alphosises
-alphyl
-alphyls
-alpinely
-alpinism
-alpinisms
-alpinist's
-als
-alsike
-alsikes
-alstroemeria
-alstroemeria's
-alstroemerias
-altaltissimo
-altaltissimos
-altarage
-altarwise
-altazimuth's
-alterabilities
-alterability
-alterability's
-alterably
-alterants
-altercative
-alterer
-alterers
-alterities
-alterity
-altern
-alternance
-alternances
-alternants
-alternat
-alternateness
-alternatenesses
-alternatim
-alternativeness
-alternativenesses
-alterne
-alternes
-altesse
-alteza
-altezza
-althorns
-altimetric
-altimetrical
-altimetrically
-altimetries
-altiplanos
-altisonant
-altitonant
-altitudinarian
-altitudinarians
-altitudinous
-altocumuli's
-altocumulus's
-altogether's
-altogethers
-altoist
-altoists
-altostrati's
-altostratus's
-altrices
-altruisms
-alts
-aludels
-alula's
-alulae
-alular
-alumin
-aluminas
-aluminates
-alumine
-alumines
-aluminic
-aluminosilicate
-aluminosilicates
-aluminothermies
-alumins
-alumish
-alumium
-alumnae's
-alumni's
-alumroot's
-alumstone
-alunites
-alure
-alvearies
-alveary
-alveated
-alveola
-alveolar's
-alveolarly
-alveolated
-alveolation
-alveolations
-alveoles
-alveoli's
-alveolitis
-alveolitis's
-alveolus's
-alway
-alycompaine
-alycompaines
-alyssum's
-ama
-amabile
-amadavat's
-amadous
-amah's
-amalgamable
-amalgamative
-amalgamator
-amalgamators
-amandines
-amanitas
-amanitin
-amanitins
-amantadine
-amantadines
-amaracus
-amaracuses
-amarant
-amarantaceous
-amarantine
-amarants
-amarelle's
-amaretti
-amarettos
-amarna
-amaryllid
-amaryllids
-amas
-amassable
-amassables
-amassers
-amassments
-amate
-amated
-amates
-amateurishnesses
-amateurisms
-amateurship
-amating
-amation
-amativeness's
-amatols
-amatorial
-amatorially
-amatorian
-amatorious
-amaurosis's
-amaurotic
-amauti
-amautik
-amautiks
-amautis
-amazedness
-amazednesses
-amazements
-amazonites
-amazonstone
-amazonstones
-ambage
-ambagitory
-amban
-ambans
-ambari
-ambaries
-ambaris
-ambassage
-ambassages
-ambassies
-ambassy
-ambatch
-ambatches
-ambeer
-ambeers
-amberfish
-amberfish's
-ambergrises
-amberies
-amberina
-amberinas
-amberite
-amberjack's
-amberoids
-amberous
-ambers
-ambery
-ambidexterities
-ambidexters
-ambidextrousness
-ambidextrousness's
-ambient's
-ambients
-ambiguousness
-ambiguousnesses
-ambilateral
-ambisexual
-ambisexualities
-ambisexuality
-ambisexuals
-ambisonic
-ambisonics
-ambit's
-ambitioned
-ambitioning
-ambitionless
-ambitiousnesses
-ambitty
-ambivalences
-ambivalencies
-ambivalency
-ambivalency's
-ambiversion's
-ambiverts
-amblings
-amblygonite's
-amblyopia's
-amblyopic
-ambo's
-amboceptors
-amboina
-amboinas
-ambones
-amboyna
-amboyna's
-amboynas
-ambrette
-ambrettes
-ambries
-ambroids
-ambrosially
-ambrosian
-ambrosias
-ambrotypes
-ambsaces
-ambulacra's
-ambulacral
-ambulacrum's
-ambulants
-ambulatorily
-ambulators
-ambuscaders
-ambuscado
-ambuscadoes
-ambuscados
-ambushment
-ambushments
-amebiasis's
-ameerate
-ameerates
-ameiosis
-amelcorn
-amelcorns
-amelia
-amelia's
-ameliorable
-ameliorant
-ameliorants
-ameliorations
-ameliorators
-amelioratory
-ameloblast
-ameloblast's
-ameloblasts
-amen's
-amenabilities
-amenableness's
-amenage
-amende
-amenders
-amene
-amened
-amener
-amenest
-amening
-amens
-amensalism
-amensalisms
-ament's
-amenta
-amentaceous
-amental
-amentia's
-amentiferous
-amentum
-amerceable
-amercer
-amercers
-amerciable
-amerciament
-amerciaments
-americiums
-amesaces
-ametabolic
-ametabolism
-ametabolous
-amethopterin
-amethopterins
-ametropia's
-ametropic
-amia
-amiabilities
-amiableness's
-amianthuses
-amiantus
-amiantuses
-amias
-amicabilities
-amicableness
-amicableness's
-amicablenesses
-amices
-amici
-amicus
-amidases
-amidic
-amidin
-amidine
-amidines
-amidins
-amidmost
-amido
-amidogen
-amidogens
-amidol
-amidols
-amidone
-amidones
-amids
-amies
-amiga
-amigas
-amildar
-amildars
-amin
-amine's
-aminic
-aminities
-aminity
-aminoaciduria
-aminoaciduria's
-aminoacidurias
-aminobutene
-aminopeptidase
-aminopeptidases
-aminophenol
-aminophenols
-aminophylline
-aminophylline's
-aminophyllines
-aminoplast's
-aminopterin
-aminopterins
-aminopyrine's
-aminotransferase
-aminotransferases
-aminotriazole
-aminotriazoles
-amins
-amirate
-amirates
-amis
-amissibilities
-amissibility
-amissible
-amissing
-amities
-amitosis's
-amitotic
-amitotically
-amitriptyline
-amitriptyline's
-amitriptylines
-amitrole
-amitroles
-amitryptyline
-amla
-amlas
-amman
-ammans
-ammine's
-ammino
-ammiral
-ammirals
-ammon
-ammonals
-ammonates
-ammoniac's
-ammoniacum
-ammonias
-ammoniation
-ammoniations
-ammonification
-ammonification's
-ammonifications
-ammonifier
-ammonifiers
-ammonite's
-ammonitic
-ammonium's
-ammono
-ammonoid
-ammonoid's
-ammonoids
-ammons
-ammophilous
-ammos
-ammunitions
-amnesias
-amnic
-amniographies
-amniography
-amnios
-amniote
-amniote's
-amniotes
-amniotomy
-amobarbital
-amobarbital's
-amobarbitals
-amoebiasis's
-amok's
-amoks
-amole
-amoles
-amomum
-amomums
-amoralism's
-amoralist's
-amoralities
-amorance
-amorant
-amorce
-amorces
-amorets
-amoretti
-amorettos
-amorini
-amornings
-amorosa
-amorosas
-amorosity
-amorosos
-amorousnesses
-amorphisms
-amorphousnesses
-amort
-amosite
-amosites
-amotion
-amotions
-amourette
-amourettes
-amove
-amoxicillin's
-amoxycillin
-amoxycillins
-ampassies
-ampassy
-amped
-ampelography
-ampelopses
-amperages
-amperometric
-amperzand
-amperzands
-amphiarthroses
-amphibia
-amphibiousness
-amphibiousnesses
-amphibole's
-amphibolies
-amphibolite's
-amphibolitic
-amphibological
-amphibology's
-amphibrach's
-amphibrachic
-amphictyonic
-amphictyons
-amphictyony's
-amphidiploid's
-amphidiploidies
-amphidiploidy
-amphigastria
-amphigastrium
-amphigory's
-amphimacers
-amphimictic
-amphimixis's
-amphioxi
-amphioxus's
-amphipathic
-amphiphile
-amphiphiles
-amphiphilic
-amphiploid
-amphiploidies
-amphiploids
-amphiploidy
-amphipod's
-amphipodous
-amphiprostyles
-amphiprotic
-amphisbaena's
-amphisbaenae
-amphisbaenic
-amphiscian
-amphiscians
-amphistomous
-amphitheatral
-amphitheatric
-amphitheatrical
-amphitheatrically
-amphithecia
-ampholyte
-ampholytes
-amphorae's
-amphoral
-amphoric
-ampicillin's
-amping
-ampleness's
-amplexus
-amplexuses
-ampliation
-ampliations
-ampliative
-amplicon
-amplicons
-amplidyne
-amplidynes
-amplifiable
-amplosome
-amplosomes
-ampulla's
-ampullaceous
-ampullae's
-ampullar
-ampullary
-ampullosity
-amputators
-amreeta
-amreetas
-amrit
-amritas
-amritattva
-amrits
-amtman
-amtmans
-amtrac
-amtrack
-amtracks
-amtracs
-amuletic
-amus
-amuser
-amusers
-amusette
-amusettes
-amusia
-amusias
-amusingness
-amusingnesses
-amusive
-amusiveness
-amygdal
-amygdala's
-amygdalaceous
-amygdales
-amygdalines
-amygdalins
-amygdaloidal
-amygdaloids
-amygdule
-amygdules
-amylases
-amylenes
-amylic
-amylogen
-amylogens
-amyloid's
-amyloidal
-amyloidoses
-amyloidosis
-amyloidosis's
-amylolysis's
-amylolytic
-amylopectins
-amyloplast
-amyloplasts
-amylopsins
-amyloses
-amyls
-amylum's
-amyotonia's
-amyotrophic
-amyotrophy
-amyotrophy's
-ana's
-anabaenas
-anabaptism
-anabaptisms
-anabaptist
-anabaptistic
-anabaptists
-anabases
-anabiosis's
-anabiotic
-anableps
-anablepses
-anabolisms
-anabolites
-anabolitic
-anabranches
-anacardium
-anacardiums
-anacatharsis
-anacathartic
-anacathartics
-anacharis
-anacharises
-anachronic
-anachronically
-anaclastic
-anaclises
-anaclisis
-anaclisis's
-anacoluthia's
-anacoluthically
-anacoluthon's
-anacreontic
-anacreontically
-anacreontics
-anacruses
-anacrustic
-anadems
-anadiplosis's
-anadyomene
-anaerobiont
-anaerobionts
-anaerobioses
-anaerobiosis
-anaerobiotic
-anaerobiotically
-anageneses
-anagenesis
-anaglyph's
-anaglyphic
-anaglypta
-anaglyptas
-anaglyptic
-anagnorises
-anagoge's
-anagogic
-anagogical
-anagogically
-anagogies
-anagogy
-anagrammatism
-anagrammatist
-anagrammatists
-anagrammatization
-anagrammatizations
-anagrammed
-anagramming
-analcime
-analcimes
-analcimic
-analcites
-analecta's
-analectic
-analects's
-analemmas
-analemmata
-analemmatic
-analeptic's
-analgesias
-analgetic
-analgetics
-analgia
-analgias
-analities
-anality
-analogic
-analogist
-analogist's
-analogists
-analogon
-analogons
-analogousnesses
-analphabet
-analphabete
-analphabetes
-analphabetic's
-analphabetism
-analphabetisms
-analphabets
-analyticities
-analyticity
-analyticity's
-analytics
-analyzabilities
-anamnesis's
-anamnestic
-anamnestically
-anamorphism's
-anamorphosis's
-anamorphous
-anan
-anana
-ananas
-ananas's
-ananases
-ananke
-anankes
-anans
-anapaestical
-anaphase's
-anaphasic
-anaphor
-anaphor's
-anaphora's
-anaphorical
-anaphorically
-anaphors
-anaphrodisia
-anaphrodisia's
-anaphrodisiacs
-anaphrodisias
-anaphylactic
-anaphylactically
-anaphylactoid
-anaphylaxis's
-anaphylaxy
-anaplasia
-anaplasia's
-anaplasias
-anaplasmoses
-anaplasmosis
-anaplasmosis's
-anaplasty's
-anaplerosis
-anaplerotic
-anaptyctic
-anaptyxes
-anarchal
-anarchial
-anarchies
-anarchisms
-anarchosyndicalism
-anarchosyndicalist
-anarchosyndicalists
-anarchs
-anarthrias
-anarthric
-anarthrously
-anarthrousness
-anasarca's
-anasarcous
-anastasis
-anastatic
-anastigmat's
-anastigmatism
-anastomosis's
-anastomotic
-anastrophe's
-anatases
-anathemata
-anathematical
-anathemitisation
-anatomical's
-anatomizations
-anatoxin
-anatoxins
-anatropies
-anatropy
-anatta
-anattas
-anattos
-anaxial
-anburies
-anbury
-ance
-ancestored
-ancestorial
-ancestoring
-ancho
-anchoresses
-anchoret
-anchoretic
-anchoretical
-anchorets
-anchorhold
-anchoritical
-anchoritically
-anchorless
-anchormen's
-anchorpeople's
-anchorwomen's
-anchos
-anchovetas
-anchovetta
-anchovettas
-anchusa
-anchusa's
-anchusas
-anchusins
-anchylosed
-anchyloses
-anchylosing
-anchylosis
-anchylosis's
-anchylostomiasis
-ancientnesses
-ancientries
-ancientry
-ancile
-ancillas
-ancipitous
-ancle
-ancles
-ancome
-ancomes
-anconal
-ancone
-anconeal
-ancones
-anconoid
-ancora
-ancress
-ancresses
-ancylostomiases
-andalusites
-andantini
-andantinos
-andesines
-andesite's
-andesitic
-andesyte
-andesytes
-andouille
-andouilles
-andouillette
-andouillettes
-andradite's
-androcentric
-androcephalous
-androdioecious
-androdioecism
-androecia
-androecial
-androgeneses
-androgenesis
-androgenesis's
-androgenetic
-androgenization
-androgenizations
-androgenize
-androgenized
-androgenizes
-androgenizing
-androgenous
-androgens
-androgyne's
-androgynies
-andrology
-andromeda
-andromeda's
-andromedas
-andromedotoxin
-andromonoecious
-andromonoecism
-androphore
-androphores
-androsterone's
-ands
-andvile
-andviles
-aneared
-anearing
-anears
-aneath
-anecdotages
-anecdotalism
-anecdotalisms
-anecdotalists
-anecdotical
-anecdotically
-anecdotist's
-anelaces
-anelastic
-anelasticities
-anelasticity
-anemogram
-anemograms
-anemographic
-anemographically
-anemographs
-anemography's
-anemometric
-anemometrical
-anemometry's
-anemophilies
-anemophily
-anemophobia
-anemophobic
-anemophobics
-anemoses
-anemosis
-anencephalia
-anencephalia's
-anencephalic
-anencephalies
-anencephaly
-anencephaly's
-anend
-anenst
-anergia
-anergias
-anergic
-anergy's
-anerly
-aneroid's
-anes
-anestrum's
-anethol
-anetholes
-anethols
-aneuploid
-aneuploidies
-aneuploids
-aneuploidy
-aneuploidy's
-aneurin's
-aneurismal
-anfractuosities
-anga
-angakok
-angakoks
-angaria
-angarias
-angaries
-angas
-angekkok
-angekkoks
-angekok
-angekoks
-angeled
-angelhood
-angelhoods
-angelicas
-angeling
-angelolatry
-angelologies
-angelologist
-angelologists
-angelophany
-angelus
-angelus's
-angeluses
-angerless
-angerly
-angico
-angicos
-anginas
-anginose
-anginous
-angiocardiographic
-angiocardiographies
-angiocardiography
-angiocarpous
-angiogeneses
-angiogram's
-angiographic
-angiography's
-angiology's
-angioma's
-angiomata
-angiomatous
-angiopathies
-angiopathy
-angiopathy's
-angiosarcoma
-angiosarcomas
-angiosarcomata
-angiospermal
-angiospermous
-angiostomatous
-angiostomous
-angiotensin's
-angiotensins
-angklung
-angklungs
-angleberries
-angleberry
-angledozer
-angledozer's
-angledozers
-anglepod
-anglepods
-anglepoises
-anglerfish
-anglerfish's
-anglerfishes
-anglesites
-anglewise
-anglice
-anglicist
-anglicists
-anglified
-anglifies
-anglify
-anglifying
-anglings
-anglist
-anglists
-anglo
-anglomania
-anglomaniac
-anglophil
-anglophil's
-anglophile's
-anglophilia
-anglophilic
-anglophils
-anglophobe
-anglophobe's
-anglophobes
-anglophobia
-anglophobic
-anglos
-angostura's
-angosturas
-angriness's
-angsts
-anguiform
-anguiped
-anguipede
-angularness
-angularnesses
-angulated
-angulately
-angulateness
-angulatenesses
-angulates
-angulating
-angulation's
-angulose
-angulous
-angustifoliate
-angustirostrate
-angwantibo's
-anharmonic
-anhedonia
-anhedonia's
-anhedonias
-anhedonic
-anhelation
-anhinga's
-anhungered
-anhungry
-anhydride's
-anhydrites
-ani's
-anicca
-aniccas
-aniconism
-aniconisms
-anicut
-anicuts
-anigh
-anight
-anil's
-anilinctus
-anilinctuses
-anilines
-anilingus
-anilinguses
-anilins
-anilities
-anima's
-animadverter
-animadverters
-animalcula
-animalcula's
-animalcular
-animalculism
-animalculist
-animalculists
-animalculum
-animalculum's
-animalic
-animalier
-animaliers
-animalism's
-animalistic
-animalists
-animality's
-animallike
-animally
-animately
-animateness
-animateness's
-animatenesses
-animater
-animaters
-animatic
-animatics
-animatingly
-animatism's
-animatisms
-animatronic
-animatronics
-animi
-animis
-animisms
-animuses
-anionic's
-aniseeds
-aniseikonia's
-aniseikonic
-anises
-anisettes
-anisic
-anisocercal
-anisodactylous
-anisogamete
-anisogamete's
-anisogametes
-anisogamic
-anisogamies
-anisogamous
-anisogamy
-anisogamy's
-anisoles
-anisometropia's
-anisometropic
-anisophyllous
-anisotropically
-anisotropism
-anisotropisms
-anker
-ankerites
-ankers
-ankled
-anklejack
-ankling
-anklong
-anklongs
-anklung
-anklungs
-ankuses
-ankush
-ankushes
-ankylosaur's
-ankylosaurus
-ankylosaurus's
-ankylosauruses
-ankylosis's
-ankylostomiases
-anlaces
-anlage's
-anlagen
-anlas
-anlases
-anna's
-annat
-annats
-annatta
-annattas
-annattos
-anneal's
-annealers
-annealing's
-annealings
-annectent
-annelidan
-annelidans
-annexational
-annexationisms
-annexationists
-annexion
-annexions
-annexment
-annexments
-annexure
-annexures
-annicut
-annicuts
-annihilable
-annihilationism
-annihilations
-annihilative
-annihilatory
-anno
-annona
-annonas
-annoyer
-annoyer's
-annoyers
-anns
-annularities
-annulars
-annulated
-annulations
-annulet's
-annulus's
-annunciative
-annunciator's
-annunciatory
-anoa's
-anobiidae
-anodally
-anodically
-anodizations
-anodynic
-anoeses
-anoestrum's
-anointers
-anointments
-anole's
-anolyte
-anolytes
-anomalistical
-anomalistically
-anomalousness's
-anomia
-anomia's
-anomias
-anomic
-anomics
-anomie's
-anon's
-anonaceous
-anonym's
-anonyma
-anonymities
-anonymousness
-anonymousnesses
-anoopsia
-anoopsias
-anopheleses
-anopheline
-anopheline's
-anophelines
-anopia
-anopia's
-anopias
-anopsia
-anopsias
-anorectal
-anoretic
-anoretics
-anorexias
-anorexies
-anorexigenic
-anorexy
-anorthite's
-anorthitic
-anorthosites
-anorthositic
-anosmia's
-anosmic
-another's
-anotherguess
-anourous
-anovulant
-anovulants
-anovular
-anovulation
-anovulation's
-anovulations
-anovulatory
-anoxia's
-anoxic
-ansa
-ansae
-ansaphone
-ansaphones
-ansated
-anserines
-anserous
-answerabilities
-answerability
-answerability's
-answerableness
-answerableness's
-answerablenesses
-answerback
-answerbacks
-answerer's
-answerless
-antae
-antalgic
-antalgics
-antalkalies
-antaphrodisiac
-antaphrodisiacs
-antar
-antara
-antarthritic
-antas
-antasthmatic
-antbear
-antbears
-antbird
-antbird's
-antbirds
-antecedences
-antecedency's
-antecedently
-antecessor
-antecessors
-antechapel
-antechapels
-antechoirs
-antedate's
-antediluvial
-antediluvially
-antediluvian's
-antediluvians
-antefix's
-antefixa
-antefixae
-antefixal
-antelucan
-antemortem
-antenatally
-antenati
-antennae's
-antennal
-antennary
-antenniferous
-antenniform
-antennular
-antennules
-antenuptial
-anteorbital
-antepasts
-antependia
-antependiums
-antepenult's
-antepenultimas
-antepenultimate's
-anteprandial
-anterior's
-anteriority's
-antetypes
-anteversions
-anteverted
-anteverting
-anteverts
-anthelia
-anthelices
-anthelions
-anthelix
-anthelminthic
-anthelminthic's
-anthelminthics
-anthelmintic's
-anthemed
-anthemia
-antheming
-anthemwise
-antheral
-antherid
-antheridia
-antheridial
-antheridium's
-antherids
-antherozoid's
-antherozooid
-antherozooids
-anthersmut
-anthesis's
-anthocarp
-anthocarpous
-anthocarps
-anthochlore
-anthocyan
-anthocyanins
-anthocyans
-anthodia
-anthoid
-anthological
-anthologizers
-anthomania
-anthomaniac
-anthomaniacs
-anthophilous
-anthophores
-anthophyllite
-anthophyllite's
-anthophyllites
-anthoxanthin
-anthozoan's
-anthozoic
-anthracenes
-anthraces
-anthracic
-anthracites
-anthracnoses
-anthracosis's
-anthranilate
-anthranilates
-anthraquinones
-anthropical
-anthropobiology
-anthropocentrically
-anthropocentricities
-anthropocentricity
-anthropocentricity's
-anthropocentrism's
-anthropogenesis's
-anthropogenetic
-anthropogenies
-anthropogeny
-anthropogeny's
-anthropogony
-anthropographic
-anthropographies
-anthropoidal
-anthropolatry's
-anthropologies
-anthropometrical
-anthropometrically
-anthropometrist
-anthropometrists
-anthropometry's
-anthropomorph
-anthropomorphisms
-anthropomorphist
-anthropomorphists
-anthropomorphite
-anthropomorphitic
-anthropomorphitism
-anthropomorphs
-anthropopathic
-anthropopathically
-anthropopathies
-anthropopathism
-anthropopathisms
-anthropophagi's
-anthropophagic
-anthropophagies
-anthropophaginian
-anthropophagite's
-anthropophagus
-anthropophagus's
-anthropophobia
-anthropophobic
-anthropophobics
-anthropophuism
-anthropophyte
-anthropopithecus
-anthropopsychic
-anthropopsychically
-anthropopsychism
-anthroposophical
-anthroposophies
-anthroposophist
-anthropotomy
-anthurium's
-antiacademic
-antiacademics
-antiaditis
-antiadministration
-antiadrenergic
-antiadrenergics
-antiaggression
-antiair
-antiaircraft's
-antiaircrafts
-antialcoholism
-antialcoholisms
-antialiasing
-antialiasings
-antialien
-antiallergenic
-antiallergenics
-antianemia
-antiantibodies
-antiantibody
-antiapartheids
-antiaphrodisiac
-antiaphrodisiacs
-antiarin
-antiarins
-antiaristocratic
-antiarrhythmic
-antiarrhythmic's
-antiarrhythmics
-antiars
-antiarthritic
-antiarthritics
-antiarthritis
-antiassimilation
-antiassimilations
-antiasthma
-antiasthmatic
-antiatom
-antiatoms
-antiauthoritarian
-antiauthoritarianism
-antiauthoritarianisms
-antiauthority
-antiauxin
-antiauxins
-antibacchius
-antibacchiuses
-antibacklash
-antiballistic
-antibarbarus
-antibarbaruses
-antibaryon's
-antibias
-antibilious
-antibillboard
-antibioses
-antibiotically
-antiblack
-antiblackism
-antiblackisms
-antiboss
-antibourgeois
-antiboycott
-antiboycotts
-antibug
-antibureaucratic
-antiburglar
-antiburglary
-antibusiness
-antibusing
-anticaking
-antically
-anticapitalism
-anticapitalisms
-anticapitalists
-anticar
-anticarcinogen
-anticarcinogenic
-anticarcinogens
-anticaries
-anticatalyst's
-anticathodes
-anticatholic
-anticellulite
-anticensorship
-antichloristic
-antichlors
-anticholinergics
-anticholinesterase
-anticholinesterases
-antichristianism
-antichristianly
-antichthones
-antichurch
-anticigarette
-anticipants
-anticipatable
-anticipatively
-anticipatorily
-anticipators
-anticity
-anticivic
-anticivism
-antick
-anticks
-anticlassical
-anticlericalisms
-anticlericalist
-anticlericalists
-anticlericals
-anticlimactical
-anticlinals
-anticling
-anticlinoriums
-anticlotting
-anticly
-anticoagulative
-anticodon
-anticodons
-anticold
-anticollision
-anticolonialisms
-anticolonialist
-anticolonialists
-anticolonials
-anticommercial
-anticommercialism
-anticommercialisms
-anticommunisms
-anticompetitive
-anticonglomerate
-anticonservation
-anticonservationist
-anticonservationists
-anticonservations
-anticonsumer
-anticonsumers
-anticonventional
-anticonvulsant
-anticonvulsant's
-anticonvulsants
-anticonvulsive
-anticonvulsives
-anticorporate
-anticorrosion
-anticorrosives
-anticorruption
-anticorruptions
-anticounterfeiting
-anticous
-anticrack
-anticreative
-anticruelty
-anticult
-anticults
-anticultural
-antidandruff
-antidefamation
-antidepression
-antidepressions
-antidepressives
-antiderivative
-antiderivatives
-antidesegregation
-antidesertification
-antidesiccant
-antidesiccants
-antidevelopment
-antidiabetic
-antidiabetic's
-antidiabetics
-antidiarrheal's
-antidilution
-antidiscrimination
-antidiuretic
-antidiuretic's
-antidiuretics
-antidogmatic
-antidora
-antidotally
-antidoted
-antidoting
-antidraft
-antidromically
-antidumping
-antieconomic
-antieducational
-antiegalitarian
-antielectron
-antielectron's
-antielectrons
-antielite
-antielites
-antielitism
-antielitisms
-antielitist
-antielitists
-antiemetic
-antiemetic's
-antiemetics
-antientropic
-antiepilepsy
-antiepileptic
-antiepileptic's
-antiepileptics
-antierotic
-antiestablishmentarian
-antiestablishmentarianism
-antiestablishmentarianisms
-antiestablishmentarians
-antiestrogen
-antiestrogens
-antievolution
-antievolutionary
-antievolutionism
-antievolutionisms
-antievolutionist
-antievolutionists
-antifamily
-antifascisms
-antifashion
-antifashionable
-antifashions
-antifat
-antifatigue
-antifebriles
-antifederal
-antifederalism
-antifederalisms
-antifederalist
-antifederalists
-antifemale
-antifeminine
-antifeminism's
-antifeminist's
-antiferromagnet
-antiferromagnetic
-antiferromagnetically
-antiferromagnetism
-antiferromagnetism's
-antiferromagnetisms
-antiferromagnets
-antifilibuster
-antifilibusters
-antiflash
-antiflu
-antifluoridationist
-antifluoridationists
-antifoam
-antifoaming
-antifog
-antifogging
-antiforeclosure
-antiforeign
-antiforeigner
-antiformalist
-antiformalists
-antifraud
-antifreezes
-antifrictions
-antifungal's
-antifur
-antigambling
-antigay
-antigene
-antigenes
-antigenically
-antigenicities
-antiglare
-antiglobulin
-antiglobulins
-antigravities
-antigrowth
-antiguerrilla
-antiguerrillas
-antigun
-antiguns
-antihalations
-antihelices
-antiheroines
-antiheroism
-antiheroisms
-antiherpes
-antihierarchical
-antihijack
-antihistaminics
-antihistorical
-antihomosexual
-antihuman
-antihumanisms
-antihumanistic
-antihumanitarian
-antihumanitarians
-antihunter
-antihunting
-antihuntings
-antihypertensive's
-antihysteric
-antihysterics
-antiinflammatory
-antijam
-antijamming
-antikickback
-antiking
-antikings
-antiknocks
-antileak
-antileft
-antilegomena
-antileprosy
-antilepton's
-antileukemic
-antiliberalism
-antiliberalisms
-antiliberals
-antilibertarian
-antilibertarians
-antilife
-antiliterate
-antiliterates
-antilitter
-antilittering
-antilog's
-antilogarithmic
-antilogical
-antilogies
-antilogous
-antilopine
-antilynching
-antimacho
-antimalaria
-antimalarial's
-antimalarials
-antimale
-antimales
-antiman
-antimanagement
-antimanagements
-antimarijuana
-antimarket
-antimask
-antimasks
-antimasques
-antimaterialism
-antimaterialisms
-antimaterialist
-antimaterialists
-antimatters
-antimechanist
-antimechanists
-antimeres
-antimerger
-antimeric
-antimetabole
-antimetaboles
-antimetabolic
-antimetabolics
-antimetabolite
-antimetabolite's
-antimetabolites
-antimetaphysical
-antimetathesis
-antimicrobial's
-antimicrobials
-antimilitarisms
-antimilitarist
-antimilitarists
-antimilitary
-antimiscegenation
-antimissile's
-antimissiles
-antimitotics
-antimnemonic
-antimnemonics
-antimodern
-antimodernist
-antimodernists
-antimoderns
-antimonarchical
-antimonarchists
-antimonate
-antimonates
-antimonials
-antimoniate
-antimoniates
-antimonide
-antimonides
-antimonies
-antimonious
-antimonite
-antimonites
-antimonopolist
-antimonopolists
-antimonopoly
-antimosquito
-antimusical
-antimutagen
-antimutagens
-antimycin
-antimycin's
-antimycins
-antinarrative
-antinational
-antinationalists
-antinatural
-antinature
-antinatures
-antineoplastic
-antineoplastic's
-antineoplastics
-antinephritic
-antinepotism
-antinepotisms
-antineutrino's
-antineutron's
-antings
-antinodal
-antinodes
-antinome
-antinomes
-antinomian's
-antinomianism
-antinomianism's
-antinomianisms
-antinomic
-antinomical
-antinomies
-antinovelist
-antinovelists
-antinovels
-antinucleons
-antinuke
-antinuker
-antinukers
-antinukes
-antiobesities
-antiobesity
-antiobscenities
-antiobscenity
-antiochianism
-antiodontalgic
-antiorganization
-antiorganizations
-antiozonant
-antiozonants
-antipapal
-antiparallels
-antiparasitics
-antiparty
-antipathetically
-antipathic
-antipathist
-antipathists
-antiperiodics
-antiperistalses
-antiperistaltic
-antiperistasis
-antipesticide
-antipetalous
-antiphiloprogenitive
-antiphlogistics
-antiphonary's
-antiphoner
-antiphoners
-antiphonic
-antiphonical
-antiphonically
-antiphony's
-antiphrasis's
-antiphrastic
-antiphrastical
-antiphrastically
-antipill
-antipills
-antipiracies
-antipiracy
-antiplague
-antiplagues
-antiplaque
-antipleasure
-antipleasures
-antipoaching
-antipodal's
-antipode's
-antipoetic
-antipoles
-antipolice
-antipolitical
-antipolitics
-antipollutionist
-antipollutionists
-antipollutions
-antipopes
-antipopular
-antiporn
-antipornographic
-antipornography
-antipot
-antipredator
-antipredators
-antipress
-antiprofiteering
-antiprogressive
-antiprostitution
-antiproton's
-antipruritic
-antipruritic's
-antipruritics
-antipsychotic's
-antipyic
-antipyics
-antipyreses
-antipyretic's
-antipyrines
-antiquarianisms
-antiquark
-antiquark's
-antiquarks
-antiquatedness
-antiquatednesses
-antiquations
-antiquenesses
-antiquers
-antiquitarian
-antiquitarians
-antirabies
-antirachitics
-antiracism
-antiracisms
-antiracist
-antiracists
-antiracketeering
-antiradar
-antiradicalism
-antiradicalisms
-antirape
-antirapes
-antirationalism
-antirationalisms
-antirationalist
-antirationalists
-antirationalities
-antirationality
-antirealism
-antirealisms
-antirealist
-antirealists
-antirecession
-antirecessionary
-antirecessions
-antired
-antireductionism
-antireductionisms
-antireductionist
-antireductionists
-antireflection
-antireflective
-antireform
-antiregulatory
-antireligion
-antirevolutionaries
-antirheumatic
-antirheumatics
-antiriot
-antiriots
-antiritualism
-antiritualisms
-antirock
-antiroll
-antiromantic
-antiromanticism
-antiromanticisms
-antiromantics
-antiroyalist
-antiroyalists
-antirust
-antirusts
-antisag
-antischizophrenia
-antischizophrenic
-antiscian
-antiscians
-antiscience
-antisciences
-antiscientific
-antiscorbutics
-antiscriptural
-antisecrecy
-antisegregation
-antiseizure
-antisense
-antisentimental
-antisepalous
-antiseparatist
-antiseparatists
-antisepses
-antisepticism
-antisex
-antisexism
-antisexisms
-antisexist
-antisexists
-antisexual
-antisexualities
-antisexuality
-antishark
-antiship
-antishock
-antishoplifting
-antiskid
-antislaveries
-antislavery's
-antisleep
-antislip
-antismog
-antismoke
-antismoker
-antismokers
-antismuggling
-antismut
-antisnob
-antisnobs
-antisocialism
-antisocialist
-antisocialists
-antisocialities
-antisociality
-antisolar
-antispast
-antispastic
-antispasts
-antispeculation
-antispeculative
-antispending
-antistat
-antistate
-antistatics
-antistats
-antistick
-antistories
-antistory
-antistress
-antistrike
-antistrophe's
-antistrophic
-antistrophically
-antistrophon
-antistrophons
-antistudent
-antisubsidy
-antisubversion
-antisubversions
-antisubversive
-antisubversives
-antisuicide
-antisyphilitic's
-antisyzygy
-antitarnish
-antitax
-antitechnological
-antitechnologies
-antitechnology
-antiterrorisms
-antiterrorists
-antithalian
-antitheism
-antitheist
-antitheistic
-antitheists
-antitheoretical
-antithet
-antithets
-antithrombin
-antithrombins
-antithyroid
-antitobacco
-antitotalitarian
-antitrade
-antitrade's
-antitraditional
-antitragi
-antitrinitarian
-antitrinitarianism
-antitruster
-antitrusters
-antitubercular
-antituberculosis
-antituberculous
-antitumoral
-antitussive's
-antitypal
-antitype's
-antityphoid
-antitypic
-antitypical
-antitypically
-antiulcer
-antiunemployment
-antiuniversities
-antiuniversity
-antiurban
-antivaccinationism
-antivaccinationist
-antivaccinationists
-antivenene
-antivenenes
-antivenom
-antivenoms
-antiviolence
-antivitamin
-antivitamins
-antivivisectionism
-antivivisections
-antiwear
-antiweed
-antiwelfare
-antiwhaling
-antiwhite
-antiwoman
-antiwrinkle
-antliae
-antliate
-antlike
-antlion's
-antoninianus
-antoninianuses
-antonomasias
-antonomastic
-antonomastically
-antonymy's
-antra
-antres
-antrorsely
-antrum's
-antthrush
-anu
-anucleate
-anucleated
-anural
-anuran's
-anureses
-anuresis
-anuresis's
-anuretic
-anuria's
-anuric
-anviltop
-anviltops
-anxiolytic
-anxiolytic's
-anxiolytics
-anxiousnesses
-anyon
-anyons
-anyroad
-anythingarian
-anythingarianism
-anythingarians
-anyu
-anyus
-anywhen
-anywhither
-anziani
-aorist's
-aoristically
-aortitis
-aortographic
-aortographies
-aortography
-aoudad's
-apache's
-apadana
-apadana's
-apage
-apagoge
-apagoges
-apagogic
-apagogical
-apagogically
-apaid
-apanaged
-aparejos
-apartheids
-apartmental
-apartnesses
-apathaton
-apathetical
-apathies
-apatites
-apatosaur's
-apatosaurus's
-apay
-apayd
-apaying
-apays
-apedom
-apeek
-apehood
-apepsia
-apepsy
-aper
-aper's
-aperient's
-aperiodicities
-aperiodicity
-aperitive
-apers
-apert
-apertness
-apery's
-aper�u's
-apeshit
-apetalies
-apetaly
-apfelstrudel
-apfelstrudels
-aphagia
-aphagia's
-aphagias
-aphakia
-aphakia's
-aphakias
-aphanipterous
-aphanite's
-aphanitic
-aphasiacs
-aphasias
-aphelia's
-apheliotropism
-apheliotropisms
-aphesis's
-aphetic
-aphetically
-aphicide
-aphicides
-aphidian
-aphidians
-aphidicide
-aphidicides
-aphidious
-apholate
-apholates
-aphonia's
-aphonics
-aphonous
-aphony
-aphorist's
-aphrodisia's
-aphrodite
-aphrodites
-aphtha
-aphthae
-aphthous
-aphyllies
-aphylly
-apiarians
-apical's
-apicals
-apiculi
-apiculture's
-apiculturist's
-apiculus
-apimania
-apimanias
-apio
-apiol
-apiologies
-apiology
-apios
-apishness
-apishnesses
-apism
-aplanat
-aplanatism
-aplanats
-aplanogamete
-aplanogametes
-aplanospores
-aplasia's
-aplastic
-aplite's
-aplitic
-aplombs
-aplustre
-aplustres
-apneuses
-apneusis
-apoapsides
-apoapsis
-apoapsis's
-apocalypticism
-apocalypticisms
-apocalyptism
-apocalyptisms
-apocalyptist
-apocalyptists
-apocarpies
-apocarps
-apocarpy
-apocatastasis
-apochromat
-apochromatism
-apochromatisms
-apochromats
-apocopated
-apocopates
-apocopating
-apocopation
-apocopations
-apocope's
-apocopic
-apocryphalnesses
-apocryphon
-apod
-apode
-apodeictical
-apodeictically
-apodes
-apodictical
-apodictically
-apodoses
-apodous
-apods
-apodyterium
-apodyteriums
-apoenzyme's
-apogamic
-apogamous
-apogamously
-apogamy's
-apogeal
-apogean
-apogeotropic
-apogeotropically
-apographs
-apolaustic
-apolipoprotein
-apolipoproteins
-apoliticality
-apoliticism
-apolline
-apollonian
-apollonicon
-apollonicons
-apollos
-apolog
-apologal
-apologiae
-apologs
-apologue's
-apolune's
-apomict's
-apomictic
-apomictical
-apomictically
-apomixes
-apomixis's
-apomorphia
-apomorphine's
-aponeurosis's
-aponeurotic
-apoop
-apophasis's
-apophatic
-apophlegmatic
-apophonies
-apophony
-apophthegm's
-apophthegmatic
-apophthegmatical
-apophthegmatically
-apophthegmatist
-apophyges
-apophyllites
-apophysate
-apophyseal
-apophysis's
-apoplectic's
-apoplectical
-apoplectics
-apoplex
-apoptoses
-aporetic
-aporias
-aposematically
-aposiopesis's
-aposiopetic
-apositia
-apositic
-aposporic
-apospories
-aposporous
-apostacies
-apostacy
-apostatic
-aposteriori
-apostille
-apostilles
-apostils
-apostlehood
-apostlehoods
-apostleships
-apostolates
-apostolical
-apostolically
-apostolicism
-apostolicities
-apostolicity
-apostrophus
-apothece
-apotheces
-apothecia's
-apothecial
-apothecium's
-apothegmatical
-apothegmatically
-apothegmatist
-apothegmatists
-apothems
-apotropaically
-apotropaism
-apotropous
-apozem
-apozems
-appair
-appalling's
-appalti
-appalto
-appanage's
-appanaged
-apparat
-apparats
-apparelment
-apparencies
-apparency
-apparency's
-apparentness
-apparentness's
-apparentnesses
-apparitors
-appartement
-appartements
-appay
-appayd
-appaying
-appays
-appealabilities
-appealability
-appealers
-appealingness
-appealingness's
-appearer
-appearers
-appeasably
-appeasingly
-appellational
-appellative's
-appellatively
-appellees
-appellor
-appellors
-appels
-appendants
-appendicectomy's
-appendicitises
-appendicle's
-appendicularian
-appendiculate
-apperception's
-apperceptive
-appercipient
-apperil
-appertainment
-appertainments
-appertinent
-appestat
-appestats
-appetence's
-appetency's
-appetent
-appetible
-appetition
-appetitions
-applaudably
-applaudingly
-applauses
-applausive
-applausively
-appleblossom
-applecart's
-applejacks
-applejohn
-applesauces
-applewife
-applewoman
-appliable
-applicabilities
-applicate
-applicatively
-appoggiatura's
-appoggiature
-appointors
-apport
-apportionable
-apportionments
-apports
-apposable
-apposer
-apposers
-appositenesses
-appositionally
-appositions
-appositively
-appraisee
-appraisements
-appraisive
-appraisively
-appreciativeness
-appreciativeness's
-appreciativenesses
-apprehensibilities
-apprehensibility
-apprehensibly
-apprehensivenesses
-apprenticehood
-apprenticement
-apprenticements
-appress
-appresses
-appressing
-appressoria
-appressorium
-appro
-appro's
-approachabilities
-approachability's
-approachableness
-approachablenesses
-approof
-approofs
-appropinquate
-appropinquated
-appropinquates
-appropinquating
-appropinquation
-appropinquity
-appropriable
-appropriatenesses
-appropriative
-appropriativeness
-approvable
-approvably
-approvance
-approvers
-approximant
-approximants
-approximative
-appui
-appuied
-appuis
-appulses
-appurtenant's
-appurtenants
-appuy
-appuyed
-appuying
-appuys
-apractic
-apraxia's
-apraxic
-apricate
-apricated
-apricates
-apricating
-aprication
-apriori
-apriorisms
-apriorist
-apriorists
-apriorities
-apriority
-apronful
-aproning
-apronstring
-aprotic
-apr�s
-apsaras
-apsarases
-apsidiole
-apsidioles
-apsis's
-apteria
-apterism
-apterium
-apteryx's
-aptitudinal
-aptitudinally
-aptnesses
-aptote
-aptotes
-aptotic
-apyrase
-apyrases
-apyrexia
-apyrexias
-aquabatic
-aquabatics
-aquaboard
-aquaboards
-aquacades
-aquacrop
-aquacrops
-aquacultural
-aquacultures
-aquaculturists
-aquadrome
-aquadromes
-aquafarm
-aquafarmed
-aquafarming
-aquafarms
-aquafer
-aquafers
-aquafortis
-aquafortist
-aquafortists
-aquamanale
-aquamanales
-aquamaniles
-aquanautics
-aquaphobe
-aquaphobes
-aquaphobia
-aquaphobia's
-aquaphobic
-aquaphobics
-aquaplaner
-aquaplaners
-aquaregia
-aquarelles
-aquarellist
-aquarellists
-aquarial
-aquarian
-aquarians
-aquariist
-aquariists
-aquarists
-aquarobic
-aquarobics
-aquatint's
-aquatinta
-aquatintas
-aquatinter
-aquatinters
-aquatintist
-aquatintists
-aquatone
-aquatones
-aquavits
-aquicultural
-aquiculturist
-aquiculturists
-aquiferous
-aquifoliaceous
-aquilegia's
-aquilinities
-aquilinity
-araba
-arabas
-arabesk
-arabesks
-arabesqued
-arabic
-arabica
-arabicas
-arabicization
-arabicizations
-arabicize
-arabicized
-arabicizes
-arabicizing
-arabilities
-arabin
-arabinoses
-arabinoside
-arabinosides
-arable's
-arables
-araceae
-arachis
-arachises
-arachnidans
-arachnoid's
-arachnoidal
-arachnoiditis
-arachnological
-arachnologist
-arachnologists
-arachnology
-arachnophobe
-arachnophobes
-arachnophobias
-arachnophobic
-araeometer
-araeometers
-araeometry
-araeosystyles
-aragonites
-aragonitic
-araise
-arak's
-aralia
-aralia's
-aralias
-arame
-arames
-aramids
-araneid
-araneids
-araneous
-arapaimas
-araponga
-arapongas
-arapunga
-arapungas
-arar
-araroba's
-arars
-araucaria's
-araucarian
-arb's
-arba
-arbalesters
-arbalests
-arbalist
-arbalister
-arbalisters
-arbalists
-arbas
-arbelest
-arbelests
-arbitrable
-arbitrarinesses
-arbitrary's
-arbitrational
-arbitrations
-arbitrative
-arbitratrix
-arbitratrixes
-arbitrement
-arbitrement's
-arbitrements
-arbitresses
-arbitrium
-arblast
-arblaster
-arblasters
-arblasts
-arboraceous
-arboreally
-arborescence
-arborescences
-arboret
-arboriculture's
-arboriculturist's
-arborio
-arborios
-arborist
-arborists
-arborous
-arboviral
-arboviruses
-arbuscle
-arbuscles
-arbute
-arbutean
-arbutes
-arcadia
-arcadian
-arcadians
-arcadias
-arcadings
-arcanas
-arcanist
-arcanists
-arcanum's
-arcatures
-arccosine
-arccosines
-archaea
-archaeans
-archaebacteria
-archaebacteria's
-archaebacterium
-archaebacterium's
-archaeoastronomer
-archaeoastronomers
-archaeoastronomical
-archaeologies
-archaeomagnetism
-archaeometallurgy
-archaeometric
-archaeometrist
-archaeometrists
-archaeopteryx's
-archaeornis's
-archaeozoologist
-archaeozoologists
-archaeozoology
-archaicism
-archaicism's
-archangelic
-archconservatives
-archdeaconates
-archdeaconry's
-archdeaconship
-archdeaconships
-archdruid
-archduchy's
-archdukedom
-archdukedoms
-archegonia's
-archegonial
-archegoniate
-archegoniates
-archegonium's
-archencephala
-archencephalon
-archencephalons
-archentera
-archenteric
-archenteron's
-archeometric
-archeometrist
-archeometrists
-archeress
-archeresses
-archerfish's
-archerfishes
-archeries
-archespore's
-archesporia
-archesporia's
-archesporial
-archesporium
-archesporium's
-archetypally
-archfelon
-archflamen
-archfoe
-archgenethliac
-archgenethliacs
-archheretic
-archicarps
-archichlamydeous
-archidiaconate
-archidiaconate's
-archidiaconates
-archiepiscopalities
-archiepiscopality
-archiepiscopally
-archiepiscopates
-archil's
-archilowe
-archimages
-archimandrite's
-archine's
-archings
-architected
-architecting
-architraved
-architype
-architypes
-archivolts
-archlet
-archlets
-archlutes
-archmock
-archnesses
-archology
-archons
-archonship
-archonships
-archontate
-archontates
-archontic
-archosaur
-archosaur's
-archosaurian
-archosaurian's
-archosaurs
-archpirate
-archpoet
-archprelate
-archpriest's
-archrivals
-archtraitor
-archvillain
-archwise
-arcings
-arckings
-arco
-arcsecond
-arcsecond's
-arcseconds
-arcsine
-arcsines
-arctangent
-arctangents
-arctically
-arctiid
-arctiid's
-arctiids
-arctoid
-arctophil
-arctophile
-arctophiles
-arctophilia
-arctophilist
-arctophilists
-arctophils
-arctophily
-arcuated
-arcuately
-arcuations
-arcubalist
-arcubalists
-arcus
-arcus's
-arcuses
-ard
-ardeb's
-ardencies
-ardri
-ardrigh
-ards
-arduousnesses
-areach
-aread
-areading
-areads
-areae
-areally
-arear
-areaway's
-areca's
-arecoline
-arecolines
-ared
-aredd
-arede
-aredes
-areding
-arefaction
-arefy
-areg
-arenation
-arenations
-arenite
-arenites
-arenose
-arenous
-areocentric
-areola's
-areolated
-areolation
-areolations
-areoles
-areosystile
-areosystiles
-arepa
-arepas
-arere
-arere's
-aret
-arethusa's
-arets
-arett
-aretted
-aretting
-aretts
-arew
-arf
-arfs
-arfvedsonite
-argal's
-argala
-argalas
-argali's
-argan
-argand
-argands
-argans
-argemone
-argemone's
-argemones
-argental
-argentine's
-argentite's
-argents
-argentums
-arghan
-arghans
-argillite's
-argils
-arginase
-arginases
-arginine's
-argle
-arglebargle
-argled
-argles
-argling
-argols
-argonaut's
-argonautic
-argons
-argotic
-argufier
-argufiers
-arguli
-argulus
-argumenta
-argumentations
-argumentativenesses
-argus
-argus's
-arguses
-argute
-argutely
-arguteness
-argyria
-argyrite
-argyrodite
-argyrodite's
-arhat
-arhats
-arhatship
-arhatships
-arhythmia
-arhythmic
-ariboflavinoses
-ariboflavinosis
-arider
-aridest
-aridisol
-aridisols
-aridities
-aridness's
-ariels
-arietta's
-ariette
-ariettes
-aril's
-ariled
-arillary
-arillate
-arillated
-arilli
-arillodes
-arilloid
-arillus
-ariosi
-arioso's
-ariot
-aripple
-aris
-arish
-arishes
-arisings
-arista's
-aristo
-aristocratical
-aristocratism
-aristology
-aristos
-arithmetics
-arithmomania
-arithmometer
-arithmometers
-arithmophobia
-arkite
-arkites
-arkoses
-arkosic
-arle
-arled
-arles
-arling
-armagnac
-armagnacs
-armamentaria
-armamentarium
-armamentarium's
-armamentariums
-armatured
-armaturing
-armer
-armers
-armet's
-armgaunt
-armguard
-armguards
-armiger's
-armigeral
-armigero
-armigeros
-armigerous
-armil
-armillae
-armillaria
-armillas
-armils
-arming's
-armlike
-armlock
-armlocks
-armoire's
-armonica
-armonicas
-armorial's
-armorially
-armorials
-armorist
-armorists
-armorplated
-armozeen
-armozeens
-armozine
-armozines
-armures
-armyworm's
-arna
-arnatto
-arnattos
-arnica's
-arnotto
-arnottos
-arnut
-arnuts
-aroba
-arobas
-aroid's
-aroint
-arointed
-arointing
-aroints
-arolla
-arollas
-aromatherapies
-aromaticities
-aromaticness
-aromaticnesses
-aromatizations
-arousals
-arouser
-arousers
-arow
-aroynt
-aroynted
-aroynting
-aroynts
-arpeggiated
-arpeggiates
-arpeggiating
-arpeggiations
-arpeggione
-arpeggiones
-arpen
-arpens
-arpent's
-arpillera
-arquebus's
-arquebusade
-arquebusades
-arquebusier
-arquebusiers
-arracacha
-arracachas
-arrack's
-arragonite
-arrah
-arrahs
-arraigner
-arraigners
-arraignings
-arrantly
-arrased
-arrasene
-arrayal
-arrayals
-arrayer
-arrayers
-arrayment
-arrayments
-arrearages
-arrect
-arreede
-arreeded
-arreedes
-arreeding
-arrestant
-arrestants
-arrestation
-arrestations
-arrestee
-arrestees
-arrester's
-arrestingly
-arrestive
-arrestments
-arrestors
-arrhenotoky
-arrhizal
-arrhythmias
-arrhythmically
-arriage
-arriages
-arriba
-arriero
-arrieros
-arrises
-arrish
-arrishes
-arrivance
-arrivederci's
-arriver
-arriver's
-arrivers
-arrivism
-arrivisme
-arrivisms
-arrivistes
-arri�r�
-arroba's
-arrogances
-arrogancies
-arrogancy
-arrogations
-arrogator
-arrogators
-arrondissements
-arrowheaded
-arrowroots
-arrowwoods
-arrowworm's
-arr�t
-arr�ts
-ars
-arsenate's
-arseniate
-arseniates
-arsenicals
-arsenics
-arsenides
-arsenites
-arseno
-arsenopyrite's
-arsenous
-arsheen
-arsheens
-arshin
-arshine
-arshines
-arshins
-arsine's
-arsino
-arsonite
-arsonites
-arsonous
-arsons
-arsphenamines
-arsyversy
-artal
-artels
-artemisia's
-arterial's
-arterially
-arterials
-arteriogram
-arteriogram's
-arteriograms
-arteriographic
-arteriographies
-arteriography
-arteriography's
-arterioscleroses
-arteriosclerotics
-arteriotomies
-arteritides
-arteritis's
-artfulnesses
-arthralgia's
-arthralgic
-arthritically
-arthrodeses
-arthrodesis
-arthrogram
-arthrogram's
-arthrograms
-arthrographies
-arthrography
-arthrography's
-arthromere's
-arthropathies
-arthropathy
-arthroplasty
-arthroplasty's
-arthropodal
-arthropodan
-arthropodous
-arthroscopically
-arthroscopy's
-arthroses
-arthrosis
-arthrospore's
-arthrotomies
-arthrotomy
-artic
-articling
-artics
-articulacies
-articularly
-articulatenesses
-articulative
-articulators
-artificialities
-artificialnesses
-artilleries
-artillerists
-artillerymen's
-artinesses
-artiodactyl's
-artiodactylous
-artisanships
-artistical
-artistries
-artlessnesses
-artocarpus
-artocarpuses
-artsman
-artwear
-arugola
-arugolas
-arugula's
-aruspices
-arval
-arvicole
-arvicoline
-arvos
-ary
-aryballoid
-aryballos
-aryballoses
-aryls
-arytaenoid's
-arytenoid's
-arytenoidal
-arythmia
-arythmias
-arythmic
-ar�te's
-asana
-asana's
-asanas
-asarabacca
-asarabacca's
-asarabaccas
-asarums
-asbestic
-asbestiform
-asbestine
-asbestosis's
-asbestous
-ascared
-ascariasis's
-ascarides
-ascarids
-ascaris
-ascendable
-ascendancies
-ascendantly
-ascenders
-ascendible
-ascensional
-ascertainably
-ascertainments
-asceses
-ascetical
-asceticisms
-aschelminth
-aschelminths
-asci's
-ascian
-ascians
-ascidia
-ascidian's
-ascites's
-ascitic
-ascitical
-ascititious
-asclepiad
-asclepiad's
-asclepiads
-asclepias
-asclepiases
-ascocarp's
-ascocarpic
-ascocarpous
-ascogonia
-ascoma
-ascoma's
-ascomata
-ascomata's
-ascomycete's
-ascomycetous
-ascorbate
-ascorbates
-ascospore's
-ascosporic
-ascosporous
-ascriptive
-ascriptively
-ascus's
-asdic's
-asea
-aseismic
-asepalous
-asepsis's
-aseptate
-aseptic's
-asepticism
-asepticisms
-aseptics
-asexualities
-ashake
-ashame
-ashamedness
-ashbin
-ashbin's
-ashcake
-ashcake's
-ashcakes
-asheries
-ashery
-ashet
-ashets
-ashfall
-ashfalls
-ashine
-ashiness
-ashinesses
-ashiver
-ashkey
-ashlared
-ashlarings
-ashler
-ashlered
-ashlering
-ashlerings
-ashlers
-ashless
-ashman
-ashmen
-ashpan
-ashpit
-ashplant
-ashplants
-ashrama
-ashramas
-ashramite
-ashramites
-askant
-askar
-askari
-askaris
-askars
-asker's
-askeses
-askesis
-askewness
-askewnesses
-asking's
-askings
-asklent
-askoi
-askos
-aslosh
-asmear
-asmoulder
-asocials
-asparaginase
-asparaginases
-asparagine's
-asparaguses
-asparkle
-aspartames
-aspartate
-aspartates
-aspectable
-asper's
-asperate
-asperated
-asperates
-asperating
-aspergation
-aspergations
-asperge
-asperged
-asperger
-aspergers
-asperges
-aspergill
-aspergill's
-aspergilla
-aspergilli
-aspergillosis's
-aspergills
-aspergillums
-asperging
-aspermia
-aspermias
-aspermic
-asperous
-asperser
-aspersers
-aspersive
-aspersively
-aspersoir
-aspersoirs
-aspersor
-aspersoria
-aspersoriums
-aspersors
-aspersory
-asphalter
-asphalters
-asphaltites
-asphaltums
-aspheric
-aspherical
-aspheterism
-asphyxial
-asphyxiants
-asphyxias
-asphyxiators
-asphyxies
-asphyxy
-aspidia
-aspidioid
-aspidium
-aspirata
-aspiratae
-aspiratory
-aspirers
-aspiringness
-aspis
-aspises
-aspish
-aspout
-asprawl
-aspread
-asprout
-asquat
-asrama
-asramas
-assagai's
-assagaied
-assagaiing
-assailableness
-assailablenesses
-assailers
-assailment
-assailments
-assais
-assart
-assarted
-assarting
-assarts
-assassinator
-assassinators
-assaulters
-assaultively
-assaultiveness
-assaultivenesses
-assayings
-assed
-assegai's
-assegaied
-assegaiing
-assemblagist
-assemblagists
-assemblance
-assemblymen's
-assemblywomen's
-assentaneous
-assentations
-assentator
-assenters
-assentient
-assentingly
-assentive
-assentiveness
-assentors
-assertable
-assertedly
-asserters
-assertible
-assertional
-assertivenesses
-assertory
-assessorial
-assessorship
-assessorships
-asseveratingly
-asseverations
-asseverative
-assibilation
-assibilations
-assiduousnesses
-assiege
-assieged
-assieges
-assieging
-assiento
-assientos
-assignabilities
-assignably
-assignats
-assignee's
-assimilabilities
-assimilability
-assimilationism
-assimilationisms
-assimilationists
-assimilations
-assimilators
-assistances
-assistantship
-assistantships
-assister
-assisters
-assistive
-assistor
-assistors
-assized
-assizer
-assizers
-assizing
-asslike
-associabilities
-associability
-associability's
-associable
-associateships
-associationism's
-associationist
-associationistic
-associationists
-associativities
-assoilment
-assoilments
-assonances
-assonated
-assonates
-assonating
-assortatively
-assortedness
-assorters
-assot
-assuagement's
-assuager
-assuagers
-assubjugate
-assuefaction
-assuefactions
-assuetude
-assuetudes
-assumabilities
-assumability
-assumer
-assumers
-assumingly
-assumings
-assumpsits
-assumptionist
-assumptionists
-assumptively
-assurable
-assuredness's
-assurers
-assurgencies
-assurgency
-asswage
-asswaged
-asswages
-asswaging
-assythment
-assythments
-astable
-astacological
-astacologist
-astacologists
-astacology
-astarboard
-astare
-astart
-astasia
-astasia's
-astasias
-astatically
-astaticism
-astaticisms
-astatines
-astatki
-asteism
-astelic
-astely
-asteria
-asterias
-asterid
-asteridian
-asteridians
-asterids
-asteriskless
-asterism's
-asterismal
-asteroidal
-asthenia's
-asthenics
-asthenies
-asthenopia's
-asthenopic
-asthenosphere's
-asthenospheric
-astheny
-astheny's
-asthmas
-asthmatical
-asthore
-asthores
-astichous
-astigmatic's
-astigmatically
-astigmatics
-astigmia's
-astilbe's
-astomous
-astone
-astonies
-astonishments
-astony
-astonying
-astoop
-astoundment
-astragal's
-astragalar
-astragali
-astragalus's
-astrakhans
-astrals
-astrand
-astraphobia's
-astrapophobia
-astricted
-astricting
-astriction
-astrictions
-astrictive
-astricts
-astringe
-astringed
-astringencies
-astringer
-astringers
-astringes
-astringing
-astrobiological
-astrobiologies
-astrobiologist
-astrobiologists
-astrobleme
-astroblemes
-astrochemist
-astrochemistries
-astrochemistry
-astrochemists
-astrocompass
-astrocompasses
-astrocyte
-astrocyte's
-astrocytes
-astrocytic
-astrocytoma
-astrocytomas
-astrocytomata
-astrodome's
-astrodynamic
-astrodynamicist
-astrodynamicists
-astrogeologic
-astrogeologies
-astrogeologist
-astrogeologists
-astroids
-astrolatry
-astrolatry's
-astrologies
-astrometeorology
-astrometric
-astrometrical
-astrometry's
-astronavigation's
-astronavigator
-astronavigators
-astronomies
-astrophel
-astrophels
-astrophotograph
-astrophotographer
-astrophotographers
-astrophotographic
-astrophotographies
-astrophotographs
-astrophysically
-astrospheres
-astrut
-astucious
-astuciously
-astucity
-astutenesses
-asudden
-asura
-asuras
-asway
-aswell
-aswim
-aswing
-aswirl
-aswoon
-asyla
-asymmetron
-asymptote's
-asynapses
-asynapsis
-asynartete
-asynartetes
-asynartetic
-asynchronies
-asynchronism's
-asynchrony
-asynchrony's
-asyndeta
-asyndetic
-asyndetically
-asyndetons
-asynergia
-asynergia's
-asynergies
-asynergy
-asynergy's
-asyntactic
-asystole
-asystole's
-asystoles
-asystolism
-at's
-atabal
-atabals
-atabeg
-atabegs
-atabek
-atabeks
-atabrin
-atacamite
-atactic
-ataghans
-atalaya
-atalayas
-atamans
-atamasco
-atamascos
-atap
-ataps
-atar
-atar's
-ataractic's
-ataraxia's
-ataraxic's
-ataraxies
-atars
-atavic
-atavisms
-ataxias
-ataxies
-ataxy
-ataxy's
-atebrin
-atechnic
-atelectasis's
-atelectatic
-ateleiosis
-ateleiosis's
-atelic
-atemoya
-atemoyas
-atemporal
-atenolol
-atenolol's
-atenolols
-ates
-athanasies
-athanasy
-athanors
-atheisms
-athelings
-athematically
-athenaeum's
-atheological
-atheology
-atheoretical
-atheous
-atherine
-atherines
-athermancies
-athermancy
-athermanous
-atherogeneses
-atherogenesis
-atherogenesis's
-atherogenic
-atherogenicities
-atherogenicity
-atheroma's
-atheromata
-atheromatoses
-atheromatosis
-atheromatous
-atheroscleroses
-atherosclerotically
-athetesis
-athetoid
-athetoids
-athetoses
-athetosic
-athetosis
-athetosis's
-athetotic
-athleticism's
-athodyd's
-athrill
-athrob
-athrocyte
-athrocytes
-athrocytoses
-athrocytosis
-athwartship
-atimy
-atlatls
-atma
-atmans
-atmas
-atmologist
-atmologists
-atmology
-atmometer's
-atmosphered
-atmospherical
-atoc
-atocia
-atocs
-atok
-atokal
-atoke
-atokes
-atokous
-atoks
-atomical
-atomicities
-atomies
-atomisms
-atomistically
-atomists
-atonable
-atonalism's
-atonalistic
-atonalists
-atonalities
-atoneable
-atonements
-atoners
-atonia
-atonia's
-atonias
-atonicities
-atonicity
-atonicity's
-atonics
-atoningly
-atony's
-atopic
-atopies
-atopy
-atopy's
-atoxic
-atpoints
-atrabiliar
-atrabiliousness
-atrabiliousnesses
-atracurium
-atrament
-atramental
-atramentous
-atraments
-atrazine
-atrazines
-atremble
-atresia
-atresia's
-atresias
-atresic
-atretic
-atria's
-atrociousnesses
-atrophia
-atrophias
-atropia
-atropines
-atropins
-atropism
-atropisms
-atropous
-attaboys
-attacher
-attachers
-attackable
-attackman
-attackmen
-attagirl
-attainabilities
-attainableness
-attainableness's
-attainablenesses
-attainders
-attainer
-attainers
-attaintment
-attaintments
-attaintures
-attap
-attaps
-attars
-attelet
-attelets
-attemptability
-attempter
-attempters
-attendancy
-attendantly
-attender's
-attending's
-attendment
-attendments
-attent
-attentat
-attentats
-attentivenesses
-attenuants
-attenuations
-attercop
-attercops
-attestable
-attestant
-attestant's
-attestants
-attestative
-attesters
-attestors
-atticisms
-atticist
-atticists
-atticized
-atticizing
-attirement
-attirements
-attirings
-attitudinally
-attitudinarians
-attollens
-attollent
-attollents
-attorneydom
-attorneyism
-attorneyship's
-attornment
-attornments
-attractance
-attractances
-attractancies
-attractancy
-attracter
-attracters
-attractingly
-attractivenesses
-attrahens
-attrahents
-attrap
-attrapped
-attrapping
-attraps
-attributers
-attributional
-attributiveness
-attributivenesses
-attributors
-attrist
-attrite
-attrited
-attrites
-attriting
-attritions
-attrits
-attritted
-attritting
-attuent
-attuite
-attuited
-attuites
-attuiting
-attuition
-attuitional
-attuitive
-attuitively
-attunements
-atwain
-atweel
-atweels
-atwixt
-atypic
-atypicalities
-atypicality
-atypicality's
-aubades
-auberges
-aubergine's
-aubergiste
-aubergistes
-aubretia
-aubretias
-aubrieta
-aubrietas
-aubrietia
-aubrietias
-auburns
-auctionable
-auctionary
-auctioneered
-auctioneering
-aucuba
-aucubas
-audaciousnesses
-audacities
-audad
-audad's
-audads
-audibilities
-audibleness
-audibleness's
-audiblenesses
-audiencia
-audients
-audile's
-auding
-audings
-audiocassette's
-audiogram
-audiogram's
-audiograms
-audiograph
-audiographs
-audiolingual
-audiologic
-audiologies
-audiometrician
-audiometricians
-audiometries
-audiometry
-audiometry's
-audiophil
-audiophils
-audiotyping
-audiotypist
-audiotypists
-audiovisual's
-audiovisually
-audiphones
-auditable
-auditee
-auditees
-auditives
-auditories
-auditorily
-auditorship
-auditorships
-auditory's
-auditress
-auditresses
-auf
-aufait
-aufgabe
-aufgabes
-aufs
-augend's
-augite's
-augitic
-augment's
-augmentable
-augmentative's
-augmentatives
-augmentor
-augmentors
-augurer
-augurers
-augurship
-augurships
-auguste
-augustnesses
-augusts
-auklet's
-aula
-aularian
-aulas
-auloi
-aumail
-aumailed
-aumailing
-aumails
-aumbries
-aumbry
-aumil
-aumils
-aune
-aunes
-aunthood
-aunthoods
-auntlier
-auntliest
-auntlike
-auntly
-aurar's
-aurate
-aurated
-aurates
-aureately
-aurei
-aureity
-aurelian
-aurelias
-aureolae
-aureoled
-aureoling
-aureomycin
-aures
-auricula's
-auriculae
-auricular's
-auricularly
-auriculated
-auriculately
-auriform
-auris
-auriscopes
-aurists
-aurochs's
-aurorally
-aurorean
-aurums
-auscultative
-auscultator
-auscultators
-auscultatory
-ausform
-ausformed
-ausforming
-ausforms
-auslander
-auslanders
-auslese
-ausleses
-auspex's
-auspiciousnesses
-austenite's
-austenitic
-austereness
-austereness's
-austerenesses
-austral's
-australes
-australite
-australopithecine
-australopithecine's
-australopithecines
-australs
-austringer
-austringers
-ausubo
-ausubos
-autacoid's
-autacoidal
-autarchical
-autarchist
-autarchists
-autarchs
-autarchy's
-autarkical
-autarkist
-autarkists
-autarky's
-autecologic
-autecological
-autecologies
-auteurisms
-auteurists
-auteurs
-authentical
-authenticities
-authorcraft
-authorings
-authorish
-authorism
-authoritarianisms
-authoritativenesses
-authorless
-authorships
-autisms
-autist
-autistically
-autistics
-autists
-autoantibodies
-autoantibody
-autobahnen
-autobus's
-autocades
-autocar
-autocarp
-autocarps
-autocars
-autocatalysis's
-autocatalytic
-autocatalytically
-autocephalies
-autocephaly
-autochanger
-autochangers
-autochthonal
-autochthonies
-autochthonism
-autochthonisms
-autochthonously
-autochthons
-autochthony
-autochthony's
-autocide
-autocides
-autoclaved
-autoclaving
-autocoid
-autocoid's
-autocoids
-autocoprophagy
-autocorrelations
-autocrime
-autocritique
-autocrosses
-autocue's
-autocues
-autocycle
-autocycles
-autodestruct
-autodestructed
-autodestructing
-autodestructs
-autodial
-autodials
-autodidactically
-autodidacticism
-autodigestion
-autodyne
-autodynes
-autoed
-autoerotically
-autoeroticism's
-autoerotism's
-autoexposure
-autoflare
-autoflares
-autofocus
-autofocuses
-autogamic
-autogamous
-autogam
<TRUNCATED>
[14/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
deleted file mode 100644
index d46111c..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultSslProtocolSocketFactory.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.util;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.UnknownHostException;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-
-import org.apache.commons.httpclient.ConnectTimeoutException;
-import org.apache.commons.httpclient.HttpClientError;
-import org.apache.commons.httpclient.params.HttpConnectionParams;
-import org.apache.commons.httpclient.protocol.ControllerThreadSocketFactory;
-import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class DefaultSslProtocolSocketFactory implements SecureProtocolSocketFactory {
- /** Log object for this class. */
- private static Logger logger = LoggerFactory.getLogger(DefaultSslProtocolSocketFactory.class);
- private SSLContext sslcontext = null;
-
- /**
- * Constructor for DefaultSslProtocolSocketFactory.
- */
- public DefaultSslProtocolSocketFactory() {
- super();
- }
-
- private static SSLContext createEasySSLContext() {
- try {
- SSLContext context = SSLContext.getInstance("TLS");
- context.init(null, new TrustManager[] { new DefaultX509TrustManager(null) }, null);
-
- return context;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new HttpClientError(e.toString());
- }
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int)
- */
- public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
- }
-
- /**
- * Attempts to get a new socket connection to the given host within the
- * given time limit.
- *
- * <p>
- * To circumvent the limitations of older JREs that do not support connect
- * timeout a controller thread is executed. The controller thread attempts
- * to create a new socket within the given limit of time. If socket
- * constructor does not return until the timeout expires, the controller
- * terminates and throws an {@link ConnectTimeoutException}
- * </p>
- *
- * @param host
- * the host name/IP
- * @param port
- * the port on the host
- * @param localAddress
- * the local host name/IP to bind the socket to
- * @param localPort
- * the port on the local machine
- * @param params
- * {@link HttpConnectionParams Http connection parameters}
- *
- * @return Socket a new socket
- *
- * @throws IOException
- * if an I/O error occurs while creating the socket
- * @throws UnknownHostException
- * if the IP address of the host cannot be determined
- * @throws ConnectTimeoutException
- * DOCUMENT ME!
- * @throws IllegalArgumentException
- * DOCUMENT ME!
- */
- public Socket createSocket(final String host, final int port, final InetAddress localAddress, final int localPort, final HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
- if (params == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
-
- int timeout = params.getConnectionTimeout();
-
- if (timeout == 0) {
- return createSocket(host, port, localAddress, localPort);
- } else {
- // To be eventually deprecated when migrated to Java 1.4 or above
- return ControllerThreadSocketFactory.createSocket(this, host, port, localAddress, localPort, timeout);
- }
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int)
- */
- public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port);
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean)
- */
- public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(socket, host, port, autoClose);
- }
-
- public boolean equals(Object obj) {
- return ((obj != null) && obj.getClass().equals(DefaultX509TrustManager.class));
- }
-
- public int hashCode() {
- return DefaultX509TrustManager.class.hashCode();
- }
-
- private SSLContext getSSLContext() {
- if (this.sslcontext == null) {
- this.sslcontext = createEasySSLContext();
- }
-
- return this.sslcontext;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultX509TrustManager.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultX509TrustManager.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultX509TrustManager.java
deleted file mode 100644
index f79aca3..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/DefaultX509TrustManager.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.util;
-
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class DefaultX509TrustManager implements X509TrustManager {
-
- /** Log object for this class. */
- private static Logger logger = LoggerFactory.getLogger(DefaultX509TrustManager.class);
- private X509TrustManager standardTrustManager = null;
-
- /**
- * Constructor for DefaultX509TrustManager.
- *
- */
- public DefaultX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
- super();
-
- TrustManagerFactory factory = TrustManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- factory.init(keystore);
-
- TrustManager[] trustmanagers = factory.getTrustManagers();
-
- if (trustmanagers.length == 0) {
- throw new NoSuchAlgorithmException("SunX509 trust manager not supported");
- }
-
- this.standardTrustManager = (X509TrustManager) trustmanagers[0];
- }
-
- public X509Certificate[] getAcceptedIssuers() {
- return this.standardTrustManager.getAcceptedIssuers();
- }
-
- public boolean isClientTrusted(X509Certificate[] certificates) {
- return true;
- // return this.standardTrustManager.isClientTrusted(certificates);
- }
-
- public boolean isServerTrusted(X509Certificate[] certificates) {
- if ((certificates != null) && logger.isDebugEnabled()) {
- logger.debug("Server certificate chain:");
-
- for (int i = 0; i < certificates.length; i++) {
- if (logger.isDebugEnabled()) {
- logger.debug("X509Certificate[" + i + "]=" + certificates[i]);
- }
- }
- }
-
- if ((certificates != null) && (certificates.length == 1)) {
- X509Certificate certificate = certificates[0];
-
- try {
- certificate.checkValidity();
- } catch (CertificateException e) {
- logger.error(e.toString());
-
- return false;
- }
-
- return true;
- } else {
- return true;
- // return this.standardTrustManager.isServerTrusted(certificates);
- }
- }
-
- @Override
- public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- // TODO Auto-generated method stub
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
deleted file mode 100644
index e537efc..0000000
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc.util;
-
-import java.util.Enumeration;
-
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-
-/**
- * Created by dongli on 11/24/15.
- */
-public class Log4jConfigurer {
- private final static String DEFAULT_PATTERN_LAYOUT = "L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n";
- private static boolean INITIALIZED = false;
-
- public static void initLogger() {
- if (!INITIALIZED && !isConfigured()) {
- org.apache.log4j.BasicConfigurator.configure(new ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT)));
- }
- INITIALIZED = true;
- }
-
- private static boolean isConfigured() {
- if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) {
- return true;
- } else {
- Enumeration<?> loggers = LogManager.getCurrentLoggers();
- while (loggers.hasMoreElements()) {
- Logger logger = (Logger) loggers.nextElement();
- if (logger.getAllAppenders().hasMoreElements())
- return true;
- }
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
----------------------------------------------------------------------
diff --git a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties b/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
deleted file mode 100644
index 7b6a3dc..0000000
--- a/jdbc/src/main/resources/org-apache-kylin-jdbc.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-driver.name=Kylin JDBC Driver
-driver.version=0.1
-product.name=Kylin
-product.version=0.1
-jdbc.compliant=true
-driver.version.major=0
-driver.version.minor=8
-database.version.major=0
-database.version.minor=8
-build.timestamp=20140918-2017
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
deleted file mode 100644
index eec7f91..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DriverTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import static org.junit.Assert.assertEquals;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Properties;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Unit test for Driver.
- */
-public class DriverTest {
-
- @Test
- public void testStatementWithMockData() throws SQLException {
- Driver driver = new DummyDriver();
-
- Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
-
- ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
- while (tables.next()) {
- for (int i = 0; i < 4; i++) {
- assertEquals("dummy", tables.getString(i + 1));
- }
- for (int i = 4; i < 10; i++) {
- assertEquals(null, tables.getString(i + 1));
- }
- }
-
- Statement state = conn.createStatement();
- ResultSet resultSet = state.executeQuery("select * from test_table");
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- assertEquals(12, metadata.getColumnType(1));
- assertEquals("varchar", metadata.getColumnTypeName(1));
- assertEquals(1, metadata.isNullable(1));
-
- while (resultSet.next()) {
- assertEquals("foo", resultSet.getString(1));
- assertEquals("bar", resultSet.getString(2));
- assertEquals("tool", resultSet.getString(3));
- }
- }
-
- @Test
- public void testPreparedStatementWithMockData() throws SQLException {
- Driver driver = new DummyDriver();
-
- Connection conn = driver.connect("jdbc:kylin://test_url/test_db", null);
- PreparedStatement state = conn.prepareStatement("select * from test_table where id=?");
- state.setInt(1, 10);
- ResultSet resultSet = state.executeQuery();
-
- ResultSetMetaData metadata = resultSet.getMetaData();
- assertEquals(12, metadata.getColumnType(1));
- assertEquals("varchar", metadata.getColumnTypeName(1));
- assertEquals(1, metadata.isNullable(1));
-
- while (resultSet.next()) {
- assertEquals("foo", resultSet.getString(1));
- assertEquals("bar", resultSet.getString(2));
- assertEquals("tool", resultSet.getString(3));
- }
- }
-
- @Ignore("require dev sandbox")
- @Test
- public void testWithCubeData() throws Exception {
- Driver driver = new Driver();
- Properties info = new Properties();
- info.put("user", "ADMIN");
- info.put("password", "KYLIN");
- Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
-
- ResultSet catalogs = conn.getMetaData().getCatalogs();
- System.out.println("CATALOGS");
- printResultSetMetaData(catalogs);
- printResultSet(catalogs);
-
- ResultSet schemas = conn.getMetaData().getSchemas();
- System.out.println("SCHEMAS");
- printResultSetMetaData(schemas);
- printResultSet(schemas);
-
- ResultSet tables = conn.getMetaData().getTables(null, null, null, null);
- System.out.println("TABLES");
- printResultSetMetaData(tables);
- printResultSet(tables);
-
- for (int j = 0; j < 3; j++) {
- Statement state = conn.createStatement();
- ResultSet resultSet = state.executeQuery("select * from test_kylin_fact");
-
- printResultSetMetaData(resultSet);
- printResultSet(resultSet);
- }
- }
-
- @Ignore("require dev sandbox")
- @Test
- public void testPreparedStatementWithCubeData() throws SQLException {
- Driver driver = new Driver();
- Properties info = new Properties();
- info.put("user", "ADMIN");
- info.put("password", "KYLIN");
- Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
-
- PreparedStatement state = conn.prepareStatement("select cal_dt, count(*) from test_kylin_fact where seller_id=? group by cal_dt");
- state.setLong(1, 10000001);
- ResultSet resultSet = state.executeQuery();
-
- printResultSetMetaData(resultSet);
- printResultSet(resultSet);
- }
-
- private void printResultSet(ResultSet rs) throws SQLException {
- ResultSetMetaData meta = rs.getMetaData();
- System.out.println("Data:");
-
- while (rs.next()) {
- StringBuilder buf = new StringBuilder();
- buf.append("[");
- for (int i = 0; i < meta.getColumnCount(); i++) {
- if (i > 0)
- buf.append(", ");
- buf.append(rs.getString(i + 1));
- }
- buf.append("]");
- System.out.println(buf);
- }
- }
-
- private void printResultSetMetaData(ResultSet rs) throws SQLException {
- ResultSetMetaData metadata = rs.getMetaData();
- System.out.println("Metadata:");
-
- for (int i = 0; i < metadata.getColumnCount(); i++) {
- String metaStr = metadata.getCatalogName(i + 1) + " " + metadata.getColumnClassName(i + 1) + " " + metadata.getColumnDisplaySize(i + 1) + " " + metadata.getColumnLabel(i + 1) + " " + metadata.getColumnName(i + 1) + " " + metadata.getColumnType(i + 1) + " " + metadata.getColumnTypeName(i + 1) + " " + metadata.getPrecision(i + 1) + " " + metadata.getScale(i + 1) + " " + metadata.getSchemaName(i + 1) + " " + metadata.getTableName(i + 1);
- System.out.println(metaStr);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
deleted file mode 100644
index 8438328..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import java.io.IOException;
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.calcite.avatica.AvaticaParameter;
-import org.apache.calcite.avatica.ColumnMetaData;
-import org.apache.calcite.avatica.ColumnMetaData.Rep;
-import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog;
-import org.apache.kylin.jdbc.KylinMeta.KMetaColumn;
-import org.apache.kylin.jdbc.KylinMeta.KMetaProject;
-import org.apache.kylin.jdbc.KylinMeta.KMetaSchema;
-import org.apache.kylin.jdbc.KylinMeta.KMetaTable;
-
-/**
- */
-public class DummyClient implements IRemoteClient {
-
- public DummyClient(KylinConnection conn) {
- }
-
- @Override
- public void connect() throws IOException {
- }
-
- @Override
- public KMetaProject retrieveMetaData(String project) throws IOException {
- List<KMetaColumn> columns = new ArrayList<KMetaColumn>();
-
- KMetaTable table = new KMetaTable("dummy", "dummy", "dummy", "dummy", columns);
- List<KMetaTable> tables = new ArrayList<KMetaTable>();
- tables.add(table);
-
- KMetaSchema schema = new KMetaSchema("dummy", "dummy", tables);
- List<KMetaSchema> schemas = new ArrayList<KMetaSchema>();
- schemas.add(schema);
-
- KMetaCatalog catalog = new KMetaCatalog("dummay", schemas);
- List<KMetaCatalog> catalogs = new ArrayList<KMetaCatalog>();
- catalogs.add(catalog);
-
- return new KMetaProject(project, catalogs);
- }
-
- @Override
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
- List<Object> data = new ArrayList<Object>();
- Object[] row = new Object[] { "foo", "bar", "tool" };
- data.add(row);
-
- List<ColumnMetaData> meta = new ArrayList<ColumnMetaData>();
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
- meta.add(ColumnMetaData.dummy(ColumnMetaData.scalar(Types.VARCHAR, "varchar", Rep.STRING), true));
-
- return new QueryResult(meta, data);
- }
-
- @Override
- public void close() throws IOException {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
deleted file mode 100644
index 18e4107..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyDriver.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-/**
- */
-public class DummyDriver extends Driver {
-
- @Override
- protected String getFactoryClassName(JdbcVersion jdbcVersion) {
- return DummyJdbcFactory.class.getName();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
deleted file mode 100644
index 7ee1884..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyJdbcFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-/**
- */
-public class DummyJdbcFactory extends KylinJdbcFactory {
-
- public DummyJdbcFactory() {
- super(4, 1);
- }
-
- @Override
- public IRemoteClient newRemoteClient(KylinConnection conn) {
- return new DummyClient(conn);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java b/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
deleted file mode 100644
index 9e4b3f6..0000000
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/SQLResonseStubTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.jdbc;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-
-import java.io.IOException;
-
-import org.apache.kylin.jdbc.json.SQLResponseStub;
-import org.junit.Test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-/**
- * Created by dongli on 1/25/16.
- */
-public class SQLResonseStubTest {
-
- @Test
- public void testReadValuePartRecognizedField() throws IOException {
- final String payload = "{ \"columnMetas\":[ { \"isNullable\":1, \"displaySize\":0, \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false }, { \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.048952730908745\", \"49.048952730908745\", \"49.048952730908745\", \"1\"
], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false }";
- final SQLResponseStub stub = new ObjectMapper().readValue(payload, SQLResponseStub.class);
- assertEquals("test_kylin_cube_with_slr_desc", stub.getCube());
- assertEquals(3451, stub.getDuration());
- assertFalse(stub.getColumnMetas().isEmpty());
- assertEquals(91, stub.getColumnMetas().get(0).getColumnType());
- assertNull(stub.getColumnMetas().get(0).getLabel());
- assertFalse(stub.getResults().isEmpty());
- assertNull(stub.getExceptionMessage());
- }
-
- @Test
- public void testReadValueWithUnrecognizedField() throws IOException {
- final String payload = "{ \"columnMetas\":[ { \"Unrecognized\":0, \"isNullable\":1, \"displaySize\":0, \"label\":\"CAL_DT\", \"name\":\"CAL_DT\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":0, \"scale\":0, \"columnType\":91, \"columnTypeName\":\"DATE\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false }, { \"isNullable\":1, \"displaySize\":10, \"label\":\"LEAF_CATEG_ID\", \"name\":\"LEAF_CATEG_ID\", \"schemaName\":null, \"catelogName\":null, \"tableName\":null, \"precision\":10, \"scale\":0, \"columnType\":4, \"columnTypeName\":\"INTEGER\", \"readOnly\":true, \"writable\":false, \"caseSensitive\":true, \"searchable\":false, \"currency\":false, \"signed\":true, \"autoIncrement\":false, \"definitelyWritable\":false } ], \"results\":[ [ \"2013-08-07\", \"32996\", \"15\", \"15\", \"Auction\", \"10000000\", \"49.0489527309
08745\", \"49.048952730908745\", \"49.048952730908745\", \"1\" ], [ \"2013-08-07\", \"43398\", \"0\", \"14\", \"ABIN\", \"10000633\", \"85.78317064220418\", \"85.78317064220418\", \"85.78317064220418\", \"1\" ] ], \"cube\":\"test_kylin_cube_with_slr_desc\", \"affectedRowCount\":0, \"isException\":false, \"exceptionMessage\":null, \"duration\":3451, \"partial\":false, \"hitCache\":false }";
- final SQLResponseStub stub = new ObjectMapper().readValue(payload, SQLResponseStub.class);
- assertEquals("test_kylin_cube_with_slr_desc", stub.getCube());
- assertEquals(3451, stub.getDuration());
- assertFalse(stub.getColumnMetas().isEmpty());
- assertEquals(91, stub.getColumnMetas().get(0).getColumnType());
- assertEquals("CAL_DT", stub.getColumnMetas().get(0).getLabel());
- assertFalse(stub.getResults().isEmpty());
- assertNull(stub.getExceptionMessage());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/.gitignore
----------------------------------------------------------------------
diff --git a/job/.gitignore b/job/.gitignore
deleted file mode 100644
index 0b42d2d..0000000
--- a/job/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/job/.settings/org.eclipse.core.resources.prefs b/job/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 04cfa2c..0000000
--- a/job/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/job/.settings/org.eclipse.jdt.core.prefs b/job/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a903301..0000000
--- a/job/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/job/.settings/org.eclipse.jdt.ui.prefs b/job/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index dece0e6..0000000
--- a/job/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/job/dependency-reduced-pom.xml b/job/dependency-reduced-pom.xml
deleted file mode 100644
index 4073e53..0000000
--- a/job/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,594 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <parent>
- <artifactId>kylin</artifactId>
- <groupId>org.apache.kylin</groupId>
- <version>0.6.4-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-job</artifactId>
- <name>Kylin:Job</name>
- <url>http://maven.apache.org</url>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <minimizeJar>true</minimizeJar>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- <exclusion>
- <artifactId>curator-client</artifactId>
- <groupId>org.apache.curator</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-math3</artifactId>
- <groupId>org.apache.commons</groupId>
- </exclusion>
- <exclusion>
- <artifactId>xmlenc</artifactId>
- <groupId>xmlenc</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-net</artifactId>
- <groupId>commons-net</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty-util</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-json</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jasper-compiler</artifactId>
- <groupId>tomcat</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jasper-runtime</artifactId>
- <groupId>tomcat</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsp-api</artifactId>
- <groupId>javax.servlet.jsp</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-el</artifactId>
- <groupId>commons-el</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jets3t</artifactId>
- <groupId>net.java.dev.jets3t</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-core-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-mapper-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>avro</artifactId>
- <groupId>org.apache.avro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>gson</artifactId>
- <groupId>com.google.code.gson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>htrace-core</artifactId>
- <groupId>org.htrace</groupId>
- </exclusion>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-compress</artifactId>
- <groupId>org.apache.commons</groupId>
- </exclusion>
- <exclusion>
- <artifactId>curator-client</artifactId>
- <groupId>org.apache.curator</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>jdk.tools</artifactId>
- <groupId>jdk.tools</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>hadoop-yarn-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>guice-servlet</artifactId>
- <groupId>com.google.inject.extensions</groupId>
- </exclusion>
- <exclusion>
- <artifactId>netty</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>avro</artifactId>
- <groupId>org.apache.avro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>2.6.0</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>hadoop-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-hdfs</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-app</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-hs</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.mrunit</groupId>
- <artifactId>mrunit</artifactId>
- <version>1.0.0</version>
- <classifier>hadoop2</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>mockito-all</artifactId>
- <groupId>org.mockito</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <version>0.98.4-hadoop2</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>hbase-hadoop-compat</artifactId>
- <groupId>org.apache.hbase</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-core</artifactId>
- <groupId>com.yammer.metrics</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>0.98.4-hadoop2</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>hbase-protocol</artifactId>
- <groupId>org.apache.hbase</groupId>
- </exclusion>
- <exclusion>
- <artifactId>htrace-core</artifactId>
- <groupId>org.cloudera.htrace</groupId>
- </exclusion>
- <exclusion>
- <artifactId>netty</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-mapper-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <version>0.98.4-hadoop2</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>hbase-prefix-tree</artifactId>
- <groupId>org.apache.hbase</groupId>
- </exclusion>
- <exclusion>
- <artifactId>high-scale-lib</artifactId>
- <groupId>com.github.stephenc.high-scale-lib</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-math</artifactId>
- <groupId>org.apache.commons</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty-sslengine</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsp-2.1</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsp-api-2.1</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api-2.5</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-jaxrs</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jamon-runtime</artifactId>
- <groupId>org.jamon</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jaxb-api</artifactId>
- <groupId>javax.xml.bind</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-client</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hbase-protocol</artifactId>
- <groupId>org.apache.hbase</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hbase-hadoop-compat</artifactId>
- <groupId>org.apache.hbase</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-core</artifactId>
- <groupId>com.yammer.metrics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty-util</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-core-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jasper-compiler</artifactId>
- <groupId>tomcat</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jasper-runtime</artifactId>
- <groupId>tomcat</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-json</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>htrace-core</artifactId>
- <groupId>org.cloudera.htrace</groupId>
- </exclusion>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-mapper-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>guice</artifactId>
- <groupId>com.google.inject</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-guice</artifactId>
- <groupId>com.sun.jersey.contribs</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jettison</artifactId>
- <groupId>org.codehaus.jettison</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-client</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-server-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>guice-servlet</artifactId>
- <groupId>com.google.inject.extensions</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jaxb-api</artifactId>
- <groupId>javax.xml.bind</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-json</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty-util</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>zookeeper</artifactId>
- <groupId>org.apache.zookeeper</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>hamcrest-core</artifactId>
- <groupId>org.hamcrest</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-model</artifactId>
- <version>3.1.1</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>plexus-utils</artifactId>
- <groupId>org.codehaus.plexus</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>2.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>xercesImpl</artifactId>
- <groupId>xerces</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsp-api</artifactId>
- <groupId>javax.servlet.jsp</groupId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>xmlenc</artifactId>
- <groupId>xmlenc</groupId>
- </exclusion>
- <exclusion>
- <artifactId>htrace-core</artifactId>
- <groupId>org.htrace</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-core-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jasper-runtime</artifactId>
- <groupId>tomcat</groupId>
- </exclusion>
- <exclusion>
- <artifactId>netty</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jetty-util</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-mapper-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-</project>
-
[05/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java b/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
deleted file mode 100644
index 3329d27..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/RowCounterCLI.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class RowCounterCLI {
- private static final Logger logger = LoggerFactory.getLogger(RowCounterCLI.class);
-
- public static void main(String[] args) throws IOException {
-
- if (args == null || args.length != 3) {
- System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]");
- }
-
- System.out.println(args[0]);
- String htableName = args[0];
- System.out.println(args[1]);
- byte[] startKey = BytesUtil.fromReadableText(args[1]);
- System.out.println(args[2]);
- byte[] endKey = BytesUtil.fromReadableText(args[2]);
-
- if (startKey == null) {
- System.out.println("startkey is null ");
- } else {
- System.out.println("startkey lenght: " + startKey.length);
- }
-
- System.out.println("start key in binary: " + Bytes.toStringBinary(startKey));
- System.out.println("end key in binary: " + Bytes.toStringBinary(endKey));
-
- Configuration conf = HBaseConfiguration.create();
-
- Scan scan = new Scan();
- scan.setCaching(1024);
- scan.setCacheBlocks(true);
- scan.setStartRow(startKey);
- scan.setStopRow(endKey);
-
- logger.info("My Scan " + scan.toString());
-
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
-
- Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
- int counter = 0;
- while (iterator.hasNext()) {
- iterator.next();
- counter++;
- if (counter % 1000 == 1) {
- System.out.println("number of rows: " + counter);
- }
- }
- System.out.println("number of rows: " + counter);
- tableInterface.close();
- conn.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java b/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
deleted file mode 100644
index 30ffb46..0000000
--- a/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.tools;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-
-import com.google.common.collect.Lists;
-
-public class UpdateHTableHostCLI {
- private static final Log logger = LogFactory.getLog(UpdateHTableHostCLI.class);
- private List<String> updatedResources = Lists.newArrayList();
- private List<String> errorMsgs = Lists.newArrayList();
-
- private List<String> htables;
- private HBaseAdmin hbaseAdmin;
- private KylinConfig kylinConfig;
- private String oldHostValue;
-
- public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
- this.htables = htables;
- this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HadoopUtil.getCurrentHBaseConfiguration());
- this.kylinConfig = KylinConfig.getInstanceFromEnv();
- }
-
- public static void main(String args[]) throws Exception {
- if (args.length < 1) {
- printUsageAndExit();
- }
-
- List<String> tableNames = getHTableNames(KylinConfig.getInstanceFromEnv());
- if (!args[0].toLowerCase().equals("-from")) {
- printUsageAndExit();
- }
- String oldHostValue = args[1].toLowerCase();
- String filterType = args[2].toLowerCase();
- if (filterType.equals("-table")) {
- tableNames = filterByTables(tableNames, Arrays.asList(args).subList(3, args.length));
- } else if (filterType.equals("-cube")) {
- tableNames = filterByCubes(tableNames, Arrays.asList(args).subList(3, args.length));
- } else if (!filterType.equals("-all")) {
- printUsageAndExit();
- }
- logger.info("These htables are needed to be updated: " + StringUtils.join(tableNames, ","));
-
- UpdateHTableHostCLI updateHTableHostCLI = new UpdateHTableHostCLI(tableNames, oldHostValue);
- updateHTableHostCLI.execute();
-
- logger.info("=================================================================");
- logger.info("Run UpdateHTableHostCLI completed;");
-
- if (!updateHTableHostCLI.updatedResources.isEmpty()) {
- logger.info("Following resources are updated successfully:");
- for (String s : updateHTableHostCLI.updatedResources) {
- logger.info(s);
- }
- } else {
- logger.warn("No resource updated.");
- }
-
- if (!updateHTableHostCLI.errorMsgs.isEmpty()) {
- logger.info("Here are the error/warning messages, you may need to check:");
- for (String s : updateHTableHostCLI.errorMsgs) {
- logger.warn(s);
- }
- } else {
- logger.info("No error or warning messages; The update succeeds.");
- }
-
- logger.info("=================================================================");
- }
-
- private static void printUsageAndExit() {
- logger.info("Usage: exec -from oldHostValue -all|-cube cubeA,cubeB|-table tableA,tableB");
- System.exit(0);
- }
-
- private static List<String> getHTableNames(KylinConfig config) {
- CubeManager cubeMgr = CubeManager.getInstance(config);
-
- ArrayList<String> result = new ArrayList<String>();
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
- for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
- String tableName = seg.getStorageLocationIdentifier();
- if (!StringUtils.isBlank(tableName)) {
- result.add(tableName);
- System.out.println("added new table: " + tableName);
- }
- }
- }
-
- for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) {
- if (ii.getStatus() == RealizationStatusEnum.READY) {
- for (IISegment seg : ii.getSegments()) {//streaming segment is never "READY"
- String tableName = seg.getStorageLocationIdentifier();
- if (!StringUtils.isBlank(tableName)) {
- result.add(tableName);
- System.out.println("added new table: " + tableName);
- }
- }
- }
- }
-
- return result;
- }
-
- private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<String> result = Lists.newArrayList();
- for (String c : cubeNames) {
- c = c.trim();
- if (c.endsWith(","))
- c = c.substring(0, c.length() - 1);
-
- CubeInstance cubeInstance = cubeManager.getCube(c);
- for (CubeSegment segment : cubeInstance.getSegments()) {
- String tableName = segment.getStorageLocationIdentifier();
- if (allTableNames.contains(tableName)) {
- result.add(tableName);
- }
- }
- }
- return result;
- }
-
- private static List<String> filterByTables(List<String> allTableNames, List<String> tableNames) {
- List<String> result = Lists.newArrayList();
- for (String t : tableNames) {
- t = t.trim();
- if (t.endsWith(","))
- t = t.substring(0, t.length() - 1);
-
- if (allTableNames.contains(t)) {
- result.add(t);
- }
- }
- return result;
- }
-
- private void updateHtable(String tableName) throws IOException {
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
- desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
-
- updatedResources.add(tableName);
- }
- }
-
- public void execute() {
- for (String htable : htables) {
- try {
- updateHtable(htable);
- } catch (IOException ex) {
- ex.printStackTrace();
- errorMsgs.add("Update HTable[" + htable + "] failed: " + ex.getMessage());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java b/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
deleted file mode 100644
index 2255b09..0000000
--- a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.execution.AbstractExecutable;
-
-/**
- * Created by qianzhou on 12/16/14.
- */
-public abstract class BaseTestExecutable extends AbstractExecutable {
-
- public BaseTestExecutable() {
- super();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
deleted file mode 100644
index e8162fe..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.lock.ZookeeperJobLock;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class BuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- if (System.getProperty("hdp.version") == null) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
- }
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof CubingJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- DeployUtil.prepareTestData("inner", "test_kylin_cube_with_slr_empty");
- String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2", };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2", };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
-
- // empty segment
- long date4 = f.parse("2050-01-01").getTime();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date3, date4));
- return result;
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2022-01-01").getTime();
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube);
- }
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
deleted file mode 100644
index fb9167b..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.lock.ZookeeperJobLock;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- *
- * @author shaoshi
- *
- */
-public class BuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String TEST_II_NAME = "test_kylin_ii";
- private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- if (System.getProperty("hdp.version") == null) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
- }
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- iiManager = IIManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- IIInstance ii = iiManager.getII(TEST_II_NAME);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
- }
- }
-
- @After
- public void after() throws Exception {
- IIInstance ii = iiManager.getII(TEST_II_NAME);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
- }
- backup();
- }
-
- @Test
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildII" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildII() throws Exception {
- clearSegment(TEST_II_NAME);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 2015-1-1
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(TEST_II_NAME, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0 && "true".equals(System.getProperty("kylin.exportHtables"))) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- export.tearDown();
- }
-
- public static void main(String[] args) throws Exception {
- BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
-
- BuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DataGenTest.java b/job/src/test/java/org/apache/kylin/job/DataGenTest.java
deleted file mode 100644
index 6cc02e2..0000000
--- a/job/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class DataGenTest extends LocalFileMetadataTestCase {
-
- @Before
- public void before() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testBasics() throws Exception {
- String content = FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "10000", "1", null, "inner");// default settings
- System.out.println(content);
- assertTrue(content.contains("FP-non GTC"));
- assertTrue(content.contains("ABIN"));
-
- DeployUtil.overrideFactTableData(content, "default.test_kylin_fact");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
deleted file mode 100644
index 550bddb..0000000
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HiveClient;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.maven.model.Model;
-import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
-import org.apache.tools.ant.filters.StringInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DeployUtil {
- @SuppressWarnings("unused")
- private static final Logger logger = LoggerFactory.getLogger(DeployUtil.class);
-
- public static void initCliWorkDir() throws IOException {
- execCliCommand("rm -rf " + getHadoopCliWorkingDir());
- execCliCommand("mkdir -p " + config().getKylinJobLogDir());
- }
-
- public static void deployMetadata() throws IOException {
- // install metadata to hbase
- ResourceTool.reset(config());
- ResourceTool.copy(KylinConfig.createInstanceFromUri(AbstractKylinTestCase.LOCALMETA_TEST_DATA), config());
-
- // update cube desc signature.
- for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
- cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeManager.getInstance(config()).updateCube(cube);
- }
- }
-
- public static void overrideJobJarLocations() {
- Pair<File, File> files = getJobJarFiles();
- File jobJar = files.getFirst();
- File coprocessorJar = files.getSecond();
-
- config().overrideKylinJobJarPath(jobJar.getAbsolutePath());
- config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath());
- }
-
- public static void deployJobJars() throws IOException {
- Pair<File, File> files = getJobJarFiles();
- File originalJobJar = files.getFirst();
- File originalCoprocessorJar = files.getSecond();
-
- String jobJarPath = config().getKylinJobJarPath();
- if (StringUtils.isEmpty(jobJarPath)) {
- throw new RuntimeException("deployJobJars cannot find job jar");
- }
-
- File targetJobJar = new File(jobJarPath);
- File jobJarRenamedAsTarget = new File(originalJobJar.getParentFile(), targetJobJar.getName());
- if (originalJobJar.equals(jobJarRenamedAsTarget) == false) {
- FileUtils.copyFile(originalJobJar, jobJarRenamedAsTarget);
- }
-
- File targetCoprocessorJar = new File(config().getCoprocessorLocalJar());
- File coprocessorJarRenamedAsTarget = new File(originalCoprocessorJar.getParentFile(), targetCoprocessorJar.getName());
- if (originalCoprocessorJar.equals(coprocessorJarRenamedAsTarget) == false) {
- FileUtils.copyFile(originalCoprocessorJar, coprocessorJarRenamedAsTarget);
- }
-
- CliCommandExecutor cmdExec = config().getCliCommandExecutor();
- cmdExec.copyFile(jobJarRenamedAsTarget.getAbsolutePath(), targetJobJar.getParent());
- cmdExec.copyFile(coprocessorJarRenamedAsTarget.getAbsolutePath(), targetCoprocessorJar.getParent());
- }
-
- private static Pair<File, File> getJobJarFiles() {
- String version;
- try {
- MavenXpp3Reader pomReader = new MavenXpp3Reader();
- Model model = pomReader.read(new FileReader("../pom.xml"));
- version = model.getVersion();
- } catch (Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
-
- File jobJar = new File("../job/target", "kylin-job-" + version + "-job.jar");
- File coprocessorJar = new File("../storage/target", "kylin-storage-" + version + "-coprocessor.jar");
- return new Pair<File, File>(jobJar, coprocessorJar);
- }
-
- private static void execCliCommand(String cmd) throws IOException {
- config().getCliCommandExecutor().execute(cmd);
- }
-
- private static String getHadoopCliWorkingDir() {
- return config().getCliWorkingDir();
- }
-
- private static KylinConfig config() {
- return KylinConfig.getInstanceFromEnv();
- }
-
- // ============================================================================
-
- static final String TABLE_CAL_DT = "edw.test_cal_dt";
- static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings";
- static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
- static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
- static final String TABLE_SITES = "edw.test_sites";
-
- static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES };
-
- public static void prepareTestData(String joinType, String cubeName) throws Exception {
-
- String factTableName = TABLE_KYLIN_FACT.toUpperCase();
- String content = null;
-
- boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
- if (!buildCubeUsingProvidedData) {
- System.out.println("build cube with random dataset");
- // data is generated according to cube descriptor and saved in resource store
- if (joinType.equalsIgnoreCase("inner")) {
- content = FactTableGenerator.generate(cubeName, "10000", "1", null, "inner");
- } else if (joinType.equalsIgnoreCase("left")) {
- content = FactTableGenerator.generate(cubeName, "10000", "0.6", null, "left");
- } else {
- throw new IllegalArgumentException("Unsupported join type : " + joinType);
- }
-
- assert content != null;
- overrideFactTableData(content, factTableName);
- } else {
- System.out.println("build cube with provided dataset");
- }
-
- duplicateFactTableData(factTableName, joinType);
- deployHiveTables();
- }
-
- public static void overrideFactTableData(String factTableContent, String factTableName) throws IOException {
- // Write to resource store
- ResourceStore store = ResourceStore.getStore(config());
-
- InputStream in = new StringInputStream(factTableContent);
- String factTablePath = "/data/" + factTableName + ".csv";
- store.deleteResource(factTablePath);
- store.putResource(factTablePath, in, System.currentTimeMillis());
- in.close();
- }
-
- public static void duplicateFactTableData(String factTableName, String joinType) throws IOException {
- // duplicate a copy of this fact table, with a naming convention with fact.csv.inner or fact.csv.left
- // so that later test cases can select different data files
- ResourceStore store = ResourceStore.getStore(config());
- InputStream in = store.getResource("/data/" + factTableName + ".csv").inputStream;
- String factTablePathWithJoinType = "/data/" + factTableName + ".csv." + joinType.toLowerCase();
- store.deleteResource(factTablePathWithJoinType);
- store.putResource(factTablePathWithJoinType, in, System.currentTimeMillis());
- in.close();
- }
-
- private static void deployHiveTables() throws Exception {
-
- MetadataManager metaMgr = MetadataManager.getInstance(config());
-
- // scp data files, use the data from hbase, instead of local files
- File temp = File.createTempFile("temp", ".csv");
- temp.createNewFile();
- for (String tablename : TABLE_NAMES) {
- tablename = tablename.toUpperCase();
-
- File localBufferFile = new File(temp.getParent() + "/" + tablename + ".csv");
- localBufferFile.createNewFile();
-
- InputStream hbaseDataStream = metaMgr.getStore().getResource("/data/" + tablename + ".csv").inputStream;
- FileOutputStream localFileStream = new FileOutputStream(localBufferFile);
- IOUtils.copy(hbaseDataStream, localFileStream);
-
- hbaseDataStream.close();
- localFileStream.close();
-
- localBufferFile.deleteOnExit();
- }
- String tableFileDir = temp.getParent();
- temp.delete();
-
- HiveClient hiveClient = new HiveClient();
-
- // create hive tables
- hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase())));
-
- // load data to hive tables
- // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir));
- }
-
- private static String generateLoadDataHql(String tableName, String tableFileDir) {
- return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase();
- }
-
- private static String[] generateCreateTableHql(TableDesc tableDesc) {
-
- String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
- StringBuilder ddl = new StringBuilder();
-
- ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
- ddl.append("(" + "\n");
-
- for (int i = 0; i < tableDesc.getColumns().length; i++) {
- ColumnDesc col = tableDesc.getColumns()[i];
- if (i > 0) {
- ddl.append(",");
- }
- ddl.append(col.getName() + " " + SqlHiveDataTypeMapping.getHiveDataType((col.getDatatype())) + "\n");
- }
-
- ddl.append(")" + "\n");
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
- ddl.append("STORED AS TEXTFILE");
-
- return new String[] { dropsql, ddl.toString() };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java b/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
deleted file mode 100644
index 56df524..0000000
--- a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- * Created by qianzhou on 12/22/14.
- */
-public class ErrorTestExecutable extends BaseTestExecutable {
-
- public ErrorTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- throw new RuntimeException("test error");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
deleted file mode 100644
index e784a41..0000000
--- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HBaseMiniclusterHelper;
-import org.apache.kylin.common.util.SSHClient;
-import org.apache.kylin.job.constant.BatchConstants;
-
-public class ExportHBaseData {
-
- KylinConfig kylinConfig;
- HTableDescriptor[] allTables;
- Configuration config;
- HBaseAdmin hbase;
- CliCommandExecutor cli;
- String exportHdfsFolder;
- String exportLocalFolderParent;
- String exportLocalFolder;
- String backupArchive;
- String tableNameBase;
- long currentTIME;
-
- public ExportHBaseData() {
- try {
- setup();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private void setup() throws IOException {
-
- KylinConfig.destoryInstance();
- System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cli = kylinConfig.getCliCommandExecutor();
-
- currentTIME = System.currentTimeMillis();
- exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/";
- exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/";
- exportLocalFolder = exportLocalFolderParent + currentTIME + "/";
- backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz";
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = metadataUrl.substring(0, cut);
- String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- HConnection conn = HBaseConnection.get(hbaseUrl);
- try {
- hbase = new HBaseAdmin(conn);
- config = hbase.getConfiguration();
- allTables = hbase.listTables();
- } catch (IOException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- public void tearDown() {
-
- // cleanup hdfs
- try {
- if (cli != null && exportHdfsFolder != null) {
- cli.execute("hadoop fs -rm -r " + exportHdfsFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- // cleanup sandbox disk
- try {
- if (cli != null && exportLocalFolder != null) {
- cli.execute("rm -r " + exportLocalFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- // delete archive file on sandbox
- try {
- if (cli != null && backupArchive != null) {
- cli.execute("rm " + backupArchive);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void exportTables() throws IOException {
- cli.execute("mkdir -p " + exportLocalFolderParent);
-
- for (HTableDescriptor table : allTables) {
- String tName = table.getNameAsString();
- if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
- continue;
-
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName);
- }
-
- cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent);
- cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME);
-
- downloadToLocal();
- }
-
- public void downloadToLocal() throws IOException {
- String localArchive = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
-
- if (kylinConfig.getRunAsRemoteCommand()) {
- SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword());
- try {
- ssh.scpFileToLocal(backupArchive, localArchive);
- } catch (Exception e) {
- e.printStackTrace();
- }
- } else {
- FileUtils.copyFile(new File(backupArchive), new File(localArchive));
- }
- }
-
- public static void main(String[] args) {
- ExportHBaseData export = new ExportHBaseData();
- try {
- export.exportTables();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- export.tearDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
deleted file mode 100644
index a224214..0000000
--- a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- * Created by qianzhou on 12/22/14.
- */
-public class FailedTestExecutable extends BaseTestExecutable {
-
- public FailedTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- return new ExecuteResult(ExecuteResult.State.FAILED, "failed");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java b/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
deleted file mode 100644
index 53d9ceb..0000000
--- a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- * Created by qianzhou on 12/31/14.
- */
-public class SelfStopExecutable extends BaseTestExecutable {
-
- public SelfStopExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- if (isDiscarded()) {
- return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
- } else {
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
deleted file mode 100644
index 57a62b9..0000000
--- a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- * Created by qianzhou on 12/22/14.
- */
-public class SucceedTestExecutable extends BaseTestExecutable {
-
- public SucceedTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
deleted file mode 100644
index 6f12d30..0000000
--- a/job/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.dataGen;
-
-import java.util.ArrayList;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Created by honma on 5/29/14.
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ColumnConfig {
- @JsonProperty("columnName")
- private String columnName;
- @JsonProperty("valueSet")
- private ArrayList<String> valueSet;
- @JsonProperty("exclusive")
- private boolean exclusive;
- @JsonProperty("asRange")
- private boolean asRange;
-
- public boolean isAsRange() {
- return asRange;
- }
-
- public void setAsRange(boolean asRange) {
- this.asRange = asRange;
- }
-
- public boolean isExclusive() {
- return exclusive;
- }
-
- public void setExclusive(boolean exclusive) {
- this.exclusive = exclusive;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public ArrayList<String> getValueSet() {
- return valueSet;
- }
-
- public void setValueSet(ArrayList<String> valueSet) {
- this.valueSet = valueSet;
- }
-
-}
[08/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
deleted file mode 100644
index 3b25ee1..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/StorageCleanupJob.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.cube;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.cmd.ICommandOutput;
-import org.apache.kylin.job.cmd.ShellCmd;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * @author ysong1
- */
-public class StorageCleanupJob extends AbstractHadoopJob {
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete");
-
- protected static final Logger log = LoggerFactory.getLogger(StorageCleanupJob.class);
-
- boolean delete = false;
-
- protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
- */
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- log.info("----- jobs args: " + Arrays.toString(args));
- try {
- options.addOption(OPTION_DELETE);
- parseOptions(options, args);
-
- log.info("options: '" + getOptionsAsString() + "'");
- log.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
- delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
-
- Configuration conf = HBaseConfiguration.create(getConf());
-
- cleanUnusedIntermediateHiveTable(conf);
- cleanUnusedHdfsFiles(conf);
- cleanUnusedHBaseTables(conf);
-
- return 0;
- } catch (Exception e) {
- e.printStackTrace(System.err);
- throw e;
- }
- }
-
- private void cleanUnusedHBaseTables(Configuration conf) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIManager iiManager = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- // get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
- String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
- HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
- List<String> allTablesNeedToBeDropped = new ArrayList<String>();
- for (HTableDescriptor desc : tableDescriptors) {
- String host = desc.getValue(IRealizationConstants.HTableTag);
- if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) {
- //only take care htables that belongs to self
- allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
- }
- }
-
- // remove every segment htable from drop list
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
- for (CubeSegment seg : cube.getSegments()) {
- String tablename = seg.getStorageLocationIdentifier();
- allTablesNeedToBeDropped.remove(tablename);
- log.info("Remove table " + tablename + " from drop list, as the table belongs to cube " + cube.getName() + " with status " + cube.getStatus());
- }
- }
-
- // remove every ii segment htable from drop list
- for (IIInstance ii : iiManager.listAllIIs()) {
- for (IISegment seg : ii.getSegments()) {
- String tablename = seg.getStorageLocationIdentifier();
- allTablesNeedToBeDropped.remove(tablename);
- log.info("Remove table " + tablename + " from drop list, as the table belongs to ii " + ii.getName() + " with status " + ii.getStatus());
- }
- }
-
- if (delete == true) {
- // drop tables
- for (String htableName : allTablesNeedToBeDropped) {
- log.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- hbaseAdmin.disableTable(htableName);
- hbaseAdmin.deleteTable(htableName);
- log.info("Deleted HBase table " + htableName);
- } else {
- log.info("HBase table" + htableName + " does not exist");
- }
- }
- } else {
- System.out.println("--------------- Tables To Be Dropped ---------------");
- for (String htableName : allTablesNeedToBeDropped) {
- System.out.println(htableName);
- }
- System.out.println("----------------------------------------------------");
- }
-
- hbaseAdmin.close();
- }
-
- private void cleanUnusedHdfsFiles(Configuration conf) throws IOException {
- JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- FileSystem fs = FileSystem.get(conf);
- List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
- // GlobFilter filter = new
- // GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
- // + "/kylin-.*");
- FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
- for (FileStatus status : fStatus) {
- String path = status.getPath().getName();
- // System.out.println(path);
- if (path.startsWith(JobInstance.JOB_WORKING_DIR_PREFIX)) {
- String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
- allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
- }
- }
-
- List<String> allJobs = executableManager.getAllJobIds();
- for (String jobId : allJobs) {
- // only remove FINISHED and DISCARDED job intermediate files
- final ExecutableState state = executableManager.getOutput(jobId).getState();
- if (!state.isFinalState()) {
- String path = JobInstance.getJobWorkingDir(jobId, engineConfig.getHdfsWorkingDirectory());
- allHdfsPathsNeedToBeDeleted.remove(path);
- log.info("Remove " + path + " from deletion list, as the path belongs to job " + jobId + " with status " + state);
- }
- }
-
- // remove every segment working dir from deletion list
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
- for (CubeSegment seg : cube.getSegments()) {
- String jobUuid = seg.getLastBuildJobID();
- if (jobUuid != null && jobUuid.equals("") == false) {
- String path = JobInstance.getJobWorkingDir(jobUuid, engineConfig.getHdfsWorkingDirectory());
- allHdfsPathsNeedToBeDeleted.remove(path);
- log.info("Remove " + path + " from deletion list, as the path belongs to segment " + seg + " of cube " + cube.getName());
- }
- }
- }
-
- if (delete == true) {
- // remove files
- for (String hdfsPath : allHdfsPathsNeedToBeDeleted) {
- log.info("Deleting hdfs path " + hdfsPath);
- Path p = new Path(hdfsPath);
- if (fs.exists(p) == true) {
- fs.delete(p, true);
- log.info("Deleted hdfs path " + hdfsPath);
- } else {
- log.info("Hdfs path " + hdfsPath + "does not exist");
- }
- }
- } else {
- System.out.println("--------------- HDFS Path To Be Deleted ---------------");
- for (String hdfsPath : allHdfsPathsNeedToBeDeleted) {
- System.out.println(hdfsPath);
- }
- System.out.println("-------------------------------------------------------");
- }
-
- }
-
- private void cleanUnusedIntermediateHiveTable(Configuration conf) throws IOException {
- int uuidLength = 36;
- final String useDatabaseHql = "USE " + KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() + ";";
- StringBuilder buf = new StringBuilder();
- buf.append("hive -e \"");
- buf.append(useDatabaseHql);
- buf.append("show tables " + "\'kylin_intermediate_*\'" + "; ");
- buf.append("\"");
-
- ShellCmd cmd = new ShellCmd(buf.toString(), null, null, null, false);
- ICommandOutput output = null;
-
- try {
- output = cmd.execute();
- } catch (JobException e) {
- e.printStackTrace();
- }
-
- if (output == null)
- return;
- String outputStr = output.getOutput();
- BufferedReader reader = new BufferedReader(new StringReader(outputStr));
- String line = null;
- List<String> allJobs = executableManager.getAllJobIds();
- List<String> allHiveTablesNeedToBeDeleted = new ArrayList<String>();
- List<String> workingJobList = new ArrayList<String>();
-
- for (String jobId : allJobs) {
- // only remove FINISHED and DISCARDED job intermediate table
- final ExecutableState state = executableManager.getOutput(jobId).getState();
-
- if (!state.isFinalState()) {
- workingJobList.add(jobId);
- log.info("Exclude intermediate hive table with job id " + jobId + " with job status " + state);
- }
- }
-
- while ((line = reader.readLine()) != null) {
- if (line.startsWith("kylin_intermediate_")) {
- boolean isNeedDel = false;
- String uuid = line.substring(line.length() - uuidLength, line.length());
- uuid = uuid.replace("_", "-");
- //Check whether it's a hive table in use
- if (allJobs.contains(uuid) && !workingJobList.contains(uuid)) {
- isNeedDel = true;
- }
-
- if (isNeedDel) {
- allHiveTablesNeedToBeDeleted.add(line);
- }
- }
- }
-
- if (delete == true) {
- buf.delete(0, buf.length());
- buf.append("hive -e \"");
- buf.append(useDatabaseHql);
- for (String delHive : allHiveTablesNeedToBeDeleted) {
- buf.append("drop table if exists " + delHive + "; ");
- log.info("Remove " + delHive + " from hive tables.");
- }
- buf.append("\"");
- cmd = new ShellCmd(buf.toString(), null, null, null, false);
-
- try {
- cmd.execute();
- } catch (JobException e) {
- e.printStackTrace();
- }
- } else {
- System.out.println("------ Intermediate Hive Tables To Be Dropped ------");
- for (String hiveTable : allHiveTablesNeedToBeDeleted) {
- System.out.println(hiveTable);
- }
- System.out.println("----------------------------------------------------");
- }
-
- if (reader != null)
- reader.close();
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateDictionaryJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateDictionaryJob.java
deleted file mode 100644
index c7369c4..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateDictionaryJob.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.dict;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.cli.DictionaryGeneratorCLI;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-
-/**
- * @author ysong1
- *
- */
-
-public class CreateDictionaryJob extends AbstractHadoopJob {
-
- private int returnCode = 0;
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_SEGMENT_NAME);
- options.addOption(OPTION_INPUT_PATH);
- parseOptions(options, args);
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME);
- String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
- String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- DictionaryGeneratorCLI.processSegment(config, cubeName, segmentName, factColumnsInputPath);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
-
- return returnCode;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new CreateDictionaryJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateInvertedIndexDictionaryJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateInvertedIndexDictionaryJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateInvertedIndexDictionaryJob.java
deleted file mode 100644
index 1d5cbfe..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/dict/CreateInvertedIndexDictionaryJob.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.dict;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-
-/**
- * @author ysong1
- *
- */
-public class CreateInvertedIndexDictionaryJob extends AbstractHadoopJob {
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_INPUT_PATH);
- parseOptions(options, args);
-
- String iiname = getOptionValue(OPTION_II_NAME);
- String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(iiname);
-
- mgr.buildInvertedIndexDictionary(ii.getFirstSegment(), factColumnsInputPath);
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new CreateInvertedIndexDictionaryJob(), args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
deleted file mode 100644
index 1dc634e..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hbase;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author ysong1
- *
- */
-public class BulkLoadJob extends AbstractHadoopJob {
-
- protected static final Logger log = LoggerFactory.getLogger(BulkLoadJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- options.addOption(OPTION_CUBE_NAME);
- parseOptions(options, args);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
- // e.g
- // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
- // end with "/"
- String input = getOptionValue(OPTION_INPUT_PATH);
-
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fs = FileSystem.get(conf);
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeMgr = CubeManager.getInstance(config);
- CubeInstance cube = cubeMgr.getCube(cubeName);
- CubeDesc cubeDesc = cube.getDescriptor();
- FsPermission permission = new FsPermission((short) 0777);
- for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
- String cfName = cf.getName();
- Path columnFamilyPath = new Path(input, cfName);
-
- // File may have already been auto-loaded (in the case of MapR DB)
- if (fs.exists(columnFamilyPath)) {
- fs.setPermission(columnFamilyPath, permission);
- }
- }
-
- String[] newArgs = new String[2];
- newArgs[0] = input;
- newArgs[1] = tableName;
-
- log.debug("Start to run LoadIncrementalHFiles");
- int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
- log.debug("End to run LoadIncrementalHFiles");
- return ret;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new BulkLoadJob(), args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
deleted file mode 100644
index 027c0ca..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hbase;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.tools.DeployCoprocessorCLI;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CreateHTableJob extends AbstractHadoopJob {
-
- protected static final Logger logger = LoggerFactory.getLogger(CreateHTableJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- options.addOption(OPTION_CUBE_NAME);
- options.addOption(OPTION_PARTITION_FILE_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- parseOptions(options, args);
-
- Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
-
- String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeMgr = CubeManager.getInstance(config);
- CubeInstance cube = cubeMgr.getCube(cubeName);
- CubeDesc cubeDesc = cube.getDescriptor();
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
- // https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html
- tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());
- tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());
-
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
-
- try {
- if (User.isHBaseSecurityEnabled(conf)) {
- // add coprocessor for bulk load
- tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
- }
-
- for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
- HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName());
- cf.setMaxVersions(1);
-
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase();
-
- switch (hbaseDefaultCC) {
- case "snappy": {
- logger.info("hbase will use snappy to compress data");
- cf.setCompressionType(Algorithm.SNAPPY);
- break;
- }
- case "lzo": {
- logger.info("hbase will use lzo to compress data");
- cf.setCompressionType(Algorithm.LZO);
- break;
- }
- case "gz":
- case "gzip": {
- logger.info("hbase will use gzip to compress data");
- cf.setCompressionType(Algorithm.GZ);
- break;
- }
- case "lz4": {
- logger.info("hbase will use lz4 to compress data");
- cf.setCompressionType(Algorithm.LZ4);
- break;
- }
- default: {
- logger.info("hbase will not user any compression codec to compress data");
-
- }
- }
-
- //if (LZOSupportnessChecker.getSupportness()) {
- // logger.info("hbase will use lzo to compress data");
- // cf.setCompressionType(Algorithm.LZO);
- // } else {
- // logger.info("hbase will not use lzo to compress data");
- // }
-
- cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
- cf.setInMemory(false);
- cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
- tableDesc.addFamily(cf);
- }
-
- byte[][] splitKeys = getSplits(conf, partitionFilePath);
-
- if (admin.tableExists(tableName)) {
- // admin.disableTable(tableName);
- // admin.deleteTable(tableName);
- throw new RuntimeException("HBase table " + tableName + " exists!");
- }
-
- DeployCoprocessorCLI.deployCoprocessor(tableDesc);
-
- admin.createTable(tableDesc, splitKeys);
- logger.info("create hbase table " + tableName + " done.");
-
- return 0;
- } catch (Exception e) {
- printUsage(options);
- e.printStackTrace(System.err);
- logger.error(e.getLocalizedMessage(), e);
- return 2;
- } finally {
- admin.close();
- }
- }
-
- @SuppressWarnings("deprecation")
- public byte[][] getSplits(Configuration conf, Path path) throws Exception {
- FileSystem fs = path.getFileSystem(conf);
- if (fs.exists(path) == false) {
- System.err.println("Path " + path + " not found, no region split, HTable will be one region");
- return null;
- }
-
- List<byte[]> rowkeyList = new ArrayList<byte[]>();
- SequenceFile.Reader reader = null;
- try {
- reader = new SequenceFile.Reader(fs, path, conf);
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
- while (reader.next(key, value)) {
- rowkeyList.add(((Text) key).copyBytes());
- }
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- } finally {
- IOUtils.closeStream(reader);
- }
-
- logger.info((rowkeyList.size() + 1) + " regions");
- logger.info(rowkeyList.size() + " splits");
- for (byte[] split : rowkeyList) {
- System.out.println(StringUtils.byteToHexString(split));
- }
-
- byte[][] retValue = rowkeyList.toArray(new byte[rowkeyList.size()][]);
- return retValue.length == 0 ? null : retValue;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new CreateHTableJob(), args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
deleted file mode 100644
index b2bed9f..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.BytesSplitter;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * @author George Song (ysong1)
- */
-public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
-
- private String tableName;
- private final CubeDesc cubeDesc;
- private final CubeSegment cubeSegment;
-
- private int columnCount;
- private int[] rowKeyColumnIndexes; // the column index on flat table
- private int[][] measureColumnIndexes; // [i] is the i.th measure related column index on flat table
-
- private List<IntermediateColumnDesc> columnList = Lists.newArrayList();
-
- public CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment) {
- this.cubeDesc = cubeDesc;
- this.cubeSegment = cubeSegment;
- parseCubeDesc();
- }
-
- /**
- * @return the cubeSegment
- */
- public CubeSegment getCubeSegment() {
- return cubeSegment;
- }
-
- // check what columns from hive tables are required, and index them
- private void parseCubeDesc() {
- int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length;
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
- if (cubeSegment == null) {
- this.tableName = "kylin_intermediate_" + cubeDesc.getName();
- } else {
- this.tableName = "kylin_intermediate_" + cubeDesc.getName() + "_" + cubeSegment.getName();
- }
-
- Map<String, Integer> dimensionIndexMap = Maps.newHashMap();
- int columnIndex = 0;
- for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived()) {
- dimensionIndexMap.put(colName(col.getCanonicalName()), columnIndex);
- columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
- columnIndex++;
- }
-
- // build index
- List<TblColRef> cuboidColumns = baseCuboid.getColumns();
- rowKeyColumnIndexes = new int[rowkeyColCount];
- for (int i = 0; i < rowkeyColCount; i++) {
- String colName = colName(cuboidColumns.get(i).getCanonicalName());
- Integer dimIdx = dimensionIndexMap.get(colName);
- if (dimIdx == null) {
- throw new RuntimeException("Can't find column " + colName);
- }
- rowKeyColumnIndexes[i] = dimIdx;
- }
-
- List<MeasureDesc> measures = cubeDesc.getMeasures();
- int measureSize = measures.size();
- measureColumnIndexes = new int[measureSize][];
- for (int i = 0; i < measureSize; i++) {
- FunctionDesc func = measures.get(i).getFunction();
- List<TblColRef> colRefs = func.getParameter().getColRefs();
- if (colRefs == null) {
- measureColumnIndexes[i] = null;
- } else {
- measureColumnIndexes[i] = new int[colRefs.size()];
- for (int j = 0; j < colRefs.size(); j++) {
- TblColRef c = colRefs.get(j);
- measureColumnIndexes[i][j] = contains(columnList, c);
- if (measureColumnIndexes[i][j] < 0) {
- measureColumnIndexes[i][j] = columnIndex;
- columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), c));
- columnIndex++;
- }
- }
- }
- }
-
- columnCount = columnIndex;
- }
-
- private int contains(List<IntermediateColumnDesc> columnList, TblColRef c) {
- for (int i = 0; i < columnList.size(); i++) {
- IntermediateColumnDesc col = columnList.get(i);
-
- if (col.isSameAs(c.getTable(), c.getName()))
- return i;
- }
- return -1;
- }
-
- // sanity check the input record (in bytes) matches what's expected
- public void sanityCheck(BytesSplitter bytesSplitter) {
- if (columnCount != bytesSplitter.getBufferSize()) {
- throw new IllegalArgumentException("Expect " + columnCount + " columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
- }
-
- // TODO: check data types here
- }
-
- public CubeDesc getCubeDesc() {
- return cubeDesc;
- }
-
- public int[] getRowKeyColumnIndexes() {
- return rowKeyColumnIndexes;
- }
-
- public int[][] getMeasureColumnIndexes() {
- return measureColumnIndexes;
- }
-
- @Override
- public String getTableName(String jobUUID) {
- return tableName + "_" + jobUUID.replace("-", "_");
- }
-
- @Override
- public List<IntermediateColumnDesc> getColumnList() {
- return columnList;
- }
-
- @Override
- public DataModelDesc getDataModel() {
- return cubeDesc.getModel();
- }
-
- @Override
- public DataModelDesc.RealizationCapacity getCapacity() {
- return cubeDesc.getModel().getCapacity();
- }
-
- private static String colName(String canonicalColName) {
- return canonicalColName.replace(".", "_");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
deleted file mode 100644
index 36cb049..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IIJoinedFlatTableDesc.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
- */
-public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
-
- private IIDesc iiDesc;
- private String tableName;
- private List<IntermediateColumnDesc> columnList = Lists.newArrayList();
- private Map<String, String> tableAliasMap;
-
- public IIJoinedFlatTableDesc(IIDesc iiDesc) {
- this.iiDesc = iiDesc;
- parseIIDesc();
- }
-
- private void parseIIDesc() {
- this.tableName = "kylin_intermediate_ii_" + iiDesc.getName();
-
- int columnIndex = 0;
- for (TblColRef col : iiDesc.listAllColumns()) {
- columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
- columnIndex++;
- }
- }
-
- @Override
- public String getTableName(String jobUUID) {
- return tableName + "_" + jobUUID.replace("-", "_");
- }
-
- public List<IntermediateColumnDesc> getColumnList() {
- return columnList;
- }
-
- @Override
- public DataModelDesc getDataModel() {
- return iiDesc.getModel();
- }
-
- @Override
- public DataModelDesc.RealizationCapacity getCapacity() {
- return DataModelDesc.RealizationCapacity.SMALL;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
deleted file mode 100644
index 86a5751..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IJoinedFlatTableDesc.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import java.util.List;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
- */
-public interface IJoinedFlatTableDesc {
-
- public String getTableName(String jobUUID);
-
- public List<IntermediateColumnDesc> getColumnList();
-
- public DataModelDesc getDataModel();
-
- public DataModelDesc.RealizationCapacity getCapacity();
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hive/IntermediateColumnDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IntermediateColumnDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/IntermediateColumnDesc.java
deleted file mode 100644
index 17c87cc..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/IntermediateColumnDesc.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
- */
-public class IntermediateColumnDesc {
- private String id;
- private TblColRef colRef;
-
- public IntermediateColumnDesc(String id, TblColRef colRef) {
- this.id = id;
- this.colRef = colRef;
- }
-
- public String getId() {
- return id;
- }
-
- public String getColumnName() {
- return colRef.getName();
- }
-
- public String getDataType() {
- return colRef.getDatatype();
- }
-
- public String getTableName() {
- return colRef.getTable();
- }
-
- public boolean isSameAs(String tableName, String columnName) {
- return colRef.isSameAs(tableName, columnName);
- }
-
- public String getCanonicalName() {
- return colRef.getCanonicalName();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
deleted file mode 100644
index a1a6f22..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/SqlHiveDataTypeMapping.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.hive;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class SqlHiveDataTypeMapping {
-
- private static final Map<String, String> sqlToHiveDataTypeMapping = new HashMap<String, String>();
-
- static {
- sqlToHiveDataTypeMapping.put("short", "smallint");
- sqlToHiveDataTypeMapping.put("long", "bigint");
- sqlToHiveDataTypeMapping.put("byte", "tinyint");
- sqlToHiveDataTypeMapping.put("datetime", "date");
- }
-
- public static String getHiveDataType(String javaDataType) {
- String hiveDataType = sqlToHiveDataTypeMapping.get(javaDataType.toLowerCase());
- if (hiveDataType == null) {
- hiveDataType = javaDataType;
- }
- return hiveDataType.toLowerCase();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
deleted file mode 100644
index 3ffc770..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author ysong1
- *
- */
-public class IIBulkLoadJob extends AbstractHadoopJob {
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- options.addOption(OPTION_II_NAME);
- parseOptions(options, args);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME);
- String input = getOptionValue(OPTION_INPUT_PATH);
- String iiname = getOptionValue(OPTION_II_NAME);
-
- FileSystem fs = FileSystem.get(getConf());
- FsPermission permission = new FsPermission((short) 0777);
- fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission);
-
- int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName });
-
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = mgr.getII(iiname);
- IISegment seg = ii.getFirstSegment();
- seg.setStorageLocationIdentifier(tableName);
- seg.setStatus(SegmentStatusEnum.READY);
- mgr.updateII(ii);
-
- return hbaseExitCode;
-
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- IIBulkLoadJob job = new IIBulkLoadJob();
- job.setConf(HadoopUtil.getCurrentHBaseConfiguration());
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
deleted file mode 100644
index c032bbc..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class IICreateHFileJob extends AbstractHadoopJob {
-
- protected static final Logger log = LoggerFactory.getLogger(IICreateHFileJob.class);
-
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- parseOptions(options, args);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
-
- setJobClasspath(job);
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
- FileOutputFormat.setOutputPath(job, output);
-
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(IICreateHFileMapper.class);
- job.setMapOutputKeyClass(ImmutableBytesWritable.class);
- job.setMapOutputValueClass(KeyValue.class);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME);
- HTable htable = new HTable(HBaseConfiguration.create(getConf()), tableName);
- HFileOutputFormat.configureIncrementalLoad(job, htable);
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- IICreateHFileJob job = new IICreateHFileJob();
- job.setConf(HadoopUtil.getCurrentHBaseConfiguration());
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileMapper.java
deleted file mode 100644
index 22d9de8..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileMapper.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.invertedindex.model.IIDesc;
-
-/**
- * @author yangli9
- */
-public class IICreateHFileMapper extends KylinMapper<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable, KeyValue> {
-
- long timestamp;
-
- @Override
- protected void setup(Context context) throws IOException, InterruptedException {
- super.publishConfiguration(context.getConfiguration());
-
- timestamp = System.currentTimeMillis();
- }
-
- @Override
- protected void map(ImmutableBytesWritable key, ImmutableBytesWritable value, Context context) throws IOException, InterruptedException {
-
- KeyValue kv = new KeyValue(key.get(), key.getOffset(), key.getLength(), //
- IIDesc.HBASE_FAMILY_BYTES, 0, IIDesc.HBASE_FAMILY_BYTES.length, //
- IIDesc.HBASE_QUALIFIER_BYTES, 0, IIDesc.HBASE_QUALIFIER_BYTES.length, //
- timestamp, Type.Put, //
- value.get(), value.getOffset(), value.getLength());
-
- context.write(key, kv);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
deleted file mode 100644
index 32d065a..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.tools.DeployCoprocessorCLI;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-
-/**
- * @author George Song (ysong1)
- */
-public class IICreateHTableJob extends AbstractHadoopJob {
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_HTABLE_NAME);
- parseOptions(options, args);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME);
- String iiName = getOptionValue(OPTION_II_NAME);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- IIManager iiManager = IIManager.getInstance(config);
- IIInstance ii = iiManager.getII(iiName);
- int sharding = ii.getDescriptor().getSharding();
-
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor cf = new HColumnDescriptor(IIDesc.HBASE_FAMILY);
- cf.setMaxVersions(1);
- //cf.setCompressionType(Algorithm.LZO);
- cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
- tableDesc.addFamily(cf);
- tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());
-
- Configuration conf = HBaseConfiguration.create(getConf());
- if (User.isHBaseSecurityEnabled(conf)) {
- // add coprocessor for bulk load
- tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
- }
-
- DeployCoprocessorCLI.deployCoprocessor(tableDesc);
-
- // drop the table first
- HBaseAdmin admin = new HBaseAdmin(conf);
- if (admin.tableExists(tableName)) {
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
- }
-
- // create table
- byte[][] splitKeys = getSplits(sharding);
- if (splitKeys.length == 0)
- splitKeys = null;
- admin.createTable(tableDesc, splitKeys);
- if (splitKeys != null) {
- for (int i = 0; i < splitKeys.length; i++) {
- System.out.println("split key " + i + ": " + BytesUtil.toHex(splitKeys[i]));
- }
- }
- System.out.println("create hbase table " + tableName + " done.");
- admin.close();
-
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- //one region for one shard
- private byte[][] getSplits(int shard) {
- byte[][] result = new byte[shard - 1][];
- for (int i = 1; i < shard; ++i) {
- byte[] split = new byte[IIKeyValueCodec.SHARD_LEN];
- BytesUtil.writeUnsigned(i, split, 0, IIKeyValueCodec.SHARD_LEN);
- result[i - 1] = split;
- }
- return result;
- }
-
- public static void main(String[] args) throws Exception {
- IICreateHTableJob job = new IICreateHTableJob();
- job.setConf(HadoopUtil.getCurrentHBaseConfiguration());
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
deleted file mode 100644
index dccc594..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsCombiner.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.HashSet;
-
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * @author yangli9
- */
-public class IIDistinctColumnsCombiner extends KylinReducer<ShortWritable, Text, ShortWritable, Text> {
-
- private Text outputValue = new Text();
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- }
-
- @Override
- public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
-
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- }
-
- for (ByteArray value : set) {
- outputValue.set(value.data);
- context.write(key, outputValue);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
deleted file mode 100644
index d82386d..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.hadoop.hive.IntermediateColumnDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- */
-public class IIDistinctColumnsJob extends AbstractHadoopJob {
- protected static final Logger log = LoggerFactory.getLogger(IIDistinctColumnsJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_TABLE_NAME);
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_OUTPUT_PATH);
- parseOptions(options, args);
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- String tableName = getOptionValue(OPTION_TABLE_NAME).toUpperCase();
- String iiName = getOptionValue(OPTION_II_NAME);
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- // ----------------------------------------------------------------------------
-
- log.info("Starting: " + job.getJobName() + " on table " + tableName);
-
- IIManager iiMgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = iiMgr.getII(iiName);
- job.getConfiguration().set(BatchConstants.TABLE_NAME, tableName);
- job.getConfiguration().set(BatchConstants.TABLE_COLUMNS, getColumns(ii));
-
- setJobClasspath(job);
-
- setupMapper();
- setupReducer(output);
-
- return waitForCompletion(job);
-
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
-
- }
-
- private String getColumns(IIInstance ii) {
- IIJoinedFlatTableDesc iiflat = new IIJoinedFlatTableDesc(ii.getDescriptor());
- StringBuilder buf = new StringBuilder();
- for (IntermediateColumnDesc col : iiflat.getColumnList()) {
- if (buf.length() > 0)
- buf.append(",");
- buf.append(col.getColumnName());
- }
- return buf.toString();
- }
-
- private void setupMapper() throws IOException {
-
- String tableName = job.getConfiguration().get(BatchConstants.TABLE_NAME);
- String[] dbTableNames = HadoopUtil.parseHiveTableName(tableName);
-
- log.info("setting hcat input format, db name {} , table name {}", dbTableNames[0], dbTableNames[1]);
-
- HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);
-
- job.setInputFormatClass(HCatInputFormat.class);
-
- job.setMapperClass(IIDistinctColumnsMapper.class);
- job.setCombinerClass(IIDistinctColumnsCombiner.class);
- job.setMapOutputKeyClass(ShortWritable.class);
- job.setMapOutputValueClass(Text.class);
- }
-
- private void setupReducer(Path output) throws IOException {
- job.setReducerClass(IIDistinctColumnsReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Text.class);
-
- FileOutputFormat.setOutputPath(job, output);
- job.getConfiguration().set(BatchConstants.OUTPUT_PATH, output.toString());
-
- job.setNumReduceTasks(1);
-
- deletePath(job.getConfiguration(), output);
- }
-
- public static void main(String[] args) throws Exception {
- IIDistinctColumnsJob job = new IIDistinctColumnsJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
deleted file mode 100644
index c7d2c05..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsMapper.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.mr.KylinMapper;
-import org.apache.kylin.common.util.Bytes;
-
-/**
- * @author yangli9
- */
-public class IIDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, ShortWritable, Text> {
-
- private ShortWritable outputKey = new ShortWritable();
- private Text outputValue = new Text();
- private HCatSchema schema = null;
- private int columnSize = 0;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
- schema = HCatInputFormat.getTableSchema(context.getConfiguration());
- columnSize = schema.getFields().size();
- }
-
- @Override
- public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {
-
- HCatFieldSchema fieldSchema = null;
- for (short i = 0; i < columnSize; i++) {
- outputKey.set(i);
- fieldSchema = schema.get(i);
- Object fieldValue = record.get(fieldSchema.getName(), schema);
- if (fieldValue == null)
- continue;
- byte[] bytes = Bytes.toBytes(fieldValue.toString());
- outputValue.set(bytes, 0, bytes.length);
- context.write(outputKey, outputValue);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
deleted file mode 100644
index 2c49d11..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsReducer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.HashSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.mr.KylinReducer;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author yangli9
- */
-public class IIDistinctColumnsReducer extends KylinReducer<ShortWritable, Text, NullWritable, Text> {
-
- private String[] columns;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.publishConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
- this.columns = conf.get(BatchConstants.TABLE_COLUMNS).split(",");
- }
-
- @Override
- public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
- String columnName = columns[key.get()];
-
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- }
-
- Configuration conf = context.getConfiguration();
- FileSystem fs = FileSystem.get(conf);
- String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
- FSDataOutputStream out = fs.create(new Path(outputPath, columnName));
-
- try {
- for (ByteArray value : set) {
- out.write(value.data);
- out.write('\n');
- }
- } finally {
- out.close();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
deleted file mode 100644
index e9d8a4a..0000000
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIFlattenHiveJob.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.JoinedFlatTable;
-import org.apache.kylin.job.cmd.ICommandOutput;
-import org.apache.kylin.job.cmd.ShellCmd;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.hadoop.hive.IJoinedFlatTableDesc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/30/14.
- */
-public class IIFlattenHiveJob extends AbstractHadoopJob {
-
- protected static final Logger log = LoggerFactory.getLogger(InvertedIndexJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
- try {
- options.addOption(OPTION_II_NAME);
- parseOptions(options, args);
-
- String iiname = getOptionValue(OPTION_II_NAME);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- IIInstance iiInstance = IIManager.getInstance(config).getII(iiname);
- IIDesc iidesc = IIDescManager.getInstance(config).getIIDesc(iiInstance.getDescName());
-
- String jobUUID = "00bf87b5-c7b5-4420-a12a-07f6b37b3187";
- JobEngineConfig engineConfig = new JobEngineConfig(config);
- IJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(iidesc);
- final String useDatabaseHql = "USE " + engineConfig.getConfig().getHiveDatabaseForIntermediateTable() + ";";
- String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobUUID);
- String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, //
- JobInstance.getJobWorkingDir(jobUUID, engineConfig.getHdfsWorkingDirectory()), jobUUID);
- String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobUUID, engineConfig);
-
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(useDatabaseHql + "\n");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- System.out.println(buf.toString());
- System.out.println("========================");
-
- ShellCmd cmd = new ShellCmd(buf.toString(), null, null, null, false);
- ICommandOutput output = cmd.execute();
- System.out.println(output.getOutput());
- System.out.println(output.getExitCode());
-
- return 0;
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
- public static void main(String[] args) throws Exception {
- IIFlattenHiveJob job = new IIFlattenHiveJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
[50/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/KEYS
----------------------------------------------------------------------
diff --git a/KEYS b/KEYS
deleted file mode 100644
index 06e5b4e..0000000
--- a/KEYS
+++ /dev/null
@@ -1,131 +0,0 @@
-This file contains the PGP keys of various developers.
-Please don't use them for email unless you have to. Their main
-purpose is code signing.
-
-Examples of importing this file in your keystore:
- gpg --import KEYS.txt
- (need pgp and other examples here)
-
-Examples of adding your key to this file:
- pgp -kxa <your name> and append it to this file.
- (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
- (gpg --list-sigs <your name>
- && gpg --armor --export <your name>) >> this file.
-
------------------------------------------------------------------------------------
-
-pub 4096R/027DC364 2015-04-10
-uid Luke Han <lu...@apache.org>
-sig 3 027DC364 2015-04-10 Luke Han <lu...@apache.org>
-sub 4096R/19BCFD87 2015-04-10
-sig 027DC364 2015-04-10 Luke Han <lu...@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFUno+8BEACxRnKkHFkA/qbYKyidAsMgvgsZZIwONlAoQE+01MKsGPbJU/DR
-zVyzD5nFskaoZ3dLTeQfKP/BJ9qfXpm0zlS4PPHkrbKOeV+SawiiOvSLYCijz6vU
-T1lbgW3yIpMkFp0zpIaZo7BpA2Q9s0qvPeZrDenu1X4IeNrUHs2QInYFDrUpyghw
-BOrt4Yrx7KmtWDPia/Hvbd5nqy9coHZO+HYzBJhXPXK5arbrCDF+NEJFcUI3VlP1
-mPF0LG/SfeVtsjPVo+Ub9HIMlwd6NnzIveSXc5FWIaweQ5ko38TGgKLJ9UXkH5C+
-c/sPOaKVAYIcX5i94BLigjxAQAg15kDr1dX7gPnjQNZdLjMRWiTf4eSN5+rWK+j+
-IeA8QIcVGz7fmhyZ8J/9Gr9TdERvMuhoV/ll8aCGAQU5ApLfgVMIK8kW1vr8dFDh
-x0tnrFZIPcNRg4K9ISR/DEFxYvSkXlxaBudIJiM5itgJ2dgltIIUX5uD8h2gi94A
-4wJfI9m1F2v84lDjv6ve+O3O/P3OPErAKaKjn8w/PHFmGfVAGLkRIV/GQx39gDOf
-QFMoGhqOdLeSIWLodg73HtpZAlVEVZ0+50cHtKqWfQ357Lw3aRJdO/WWrcDZQxut
-lTiULde+vxgwWExpZf98HbpcwFNckA6+m0bwCCc9GEXkDUbPbuvpPvjJOQARAQAB
-tB1MdWtlIEhhbiA8bHVrZWhhbkBhcGFjaGUub3JnPokCOAQTAQIAIgUCVSej7wIb
-AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQoRMvEgJ9w2RoDhAArSo9kHlQ
-e84hP//+4VSccubHudPTX/+GdDcrAa0+QVS+lhnqe47sLsrvY2Hslm77AIZnYDOk
-U76FpKkK4AcMjP+ayLT6KI/ds4k9ca3OvIgj9TYtFczOg2w1dq0sTKx5MXPXJpDL
-o/Ee7Vuk10rKK9796B8keUHfFvexgfIFW39LaIOG0US5sx7IYNMIbpzEknl9dPi/
-/Mxa0+TTbeh+Jj+SkeGOV9SlcT3HWX+oXK3+saV5ChQHR4Db+wh0vhfgixi0YMbJ
-iik0aPeATiLWvccpS3t5mBE5jP+Ss1xeptXaAmXJl9QC3J5lnIMZPhI7ie83agO8
-ZRcffUMfWw/4HJbB6kj9Cs82zIT8Qfz+w+3PyoLvU65IHmY8FxknQX2jg9XFMZ+C
-c9NgJQyAOl2watglEJytjZ9UMfg36Zg0NYUmK+TMJzuMjtPbXgS7EyboMju9cdbS
-1U4AStGFHUUcnw9r7jP9O9w7DtIjo9fdIDsfHkuNf/RMbZ4xAXWF/tbk++NpIjSY
-qBXD2ZRUg+E6QfDvXOabpJ/14cVoYUlcFYeN9MpgDAKh94xmsyyxpnnyCWI3WTBj
-aMT/p7xCCvYJ5XmyyV4zRHpQ/tHuZ/XnNfkeJavCxMXuslf5AyBP64uRB00EksDf
-eMShtxE026rc9XyGf/pb7Ofg1zpbGl+Fhca5Ag0EVSej7wEQAKykheo7Hpq+m5Pl
-NFGPp3Y8kN5hV5XoJLvKgbF4UnSB8/4IXRyLfTO+5F1ePNRiRBGtWAseWKtsN/qj
-dgkc103Ckhl3NX/bH7o7mXsyOUThxAFCz/hzR1ca20H3lv63fB1L6wUD24aZ+m+9
-xf3RqiPNC/OYEvIN0NKeFNzpWWGNTaYcHReKiwwT8VpL0HzI8iu7oJI3Z/8Cmugk
-N39FiR8lJTQspmRzUJerQHqB2E6M0B252DTD7syZZYHOgP56A9V9KsMApQ4mC/qz
-ntdc0wTwLdAejrNs+qaSuIxSXhW9kPPJbazsDnvqEI3oAIU/4Zo+xt/QZpENgZTO
-xEnzIoxby75Qq54vCZSxnVsjJfJv4ZB3DIq0bwEeOK75gcyGAShnxDCzdD0rkvwj
-oyqiGu15bRKW+AiJAll2qs7PCV7sEUFx9pvz9QOx2JYyNKY246koe0j5nUDPY5uU
-NnFdq2x+LpmuT3hZ/ptEhXzgQUEyw0LFGz8sMtkZ8Vk0gbJnv1nVqYVDjfcv9Ggy
-DePBFkIpp/jWEVPWhlgQwYouNHF+W5Rs/r7eKpIwAQMw+aOcmn7BCxLkznZCvodG
-373kIjy5to+hg0tIWiVcirGdCh1ktPxeRrvTkoFd001x8GU+z+E6JGYrIIuTi4JE
-/cEqaV+wuKKqr2cFxxql9RXGYUWhABEBAAGJAh8EGAECAAkFAlUno+8CGwwACgkQ
-oRMvEgJ9w2R2VQ/9FGp/O1F8TYITJWAyOUQONdk8pZriAk+9Oon9DkxhsYdDD1oB
-td43wvFLr5dC7j9wvbnmYDWTlAwbeZYfZSS38YlrAH8/jwsuEtz6ez2faqb4qAz6
-OSGuJv3sp1T4fFUpd/O5KyGej90fmoFygSD7PdY9OEPKCHZpJuVqYMA+IaU2QXyL
-giX4jspR5SJVaJJ+kyB7+IVIVwMqxhDQ0PU4v+7hx+kVyoL7f26P+SvMkPYzM2/w
-HQrWMvuVT1NhPclk9wxdOJL1WJLNoR5U/wtZ/DBIV2gu457AyzuopHB8ye55J3q3
-mbrf24LrCUZxiofh+5GTjlC8EAcs79mAKRXUxVRah4rZovRg5jwCHgMsU0VFBzNH
-fUq0euFIhyW14c8KOhVVCUcsDwONK0EEKJnG+cPhyjrzhdtfEy6StDCvg43PI7/w
-Dodm8LJi7H1yG4LHDKalZ22st2K5DjYJWRBWXUYpCFWxQiOzGq8LdyYL3lQ/lBxt
-ElBcVmy0Qvqf5/r9jVOpZWWvoAhJT0bj5B0Ca3CuGtkiVPJIMc2QuTAItWyRDN7T
-EBG4pxGI2XsApbts0AkkINEYAKN+Z/o9BtrpEvodFPn2TBR3D9zPM+XiSZXTZ/gG
-zJ5Mba6u6w6DZa238CtPdRTMh5ZAGhgkWYCrpQJ+CD9qflqNFzpTQ4FtPg8=
-=YeBe
------END PGP PUBLIC KEY BLOCK-----
-pub 4096R/6FCF4720 2015-09-29
-uid [ultimate] Shaofeng Shi <sh...@apache.org>
-sig 3 6FCF4720 2015-09-29 Shaofeng Shi <sh...@apache.org>
-sub 4096R/90EFBBD8 2015-09-29
-sig 6FCF4720 2015-09-29 Shaofeng Shi <sh...@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Comment: GPGTools - https://gpgtools.org
-
-mQINBFYKTLQBEACwI2aNrCpaYAHI4nMjKSVOlBBCVA5mPaLTLeRXPWm9LY1HSjtl
-HAS3GZYijhwhJ9ZiDtHTvZUyZdEAJZf16jNg7p5dhdSfjp/b91a8NfNuoLuSXx7R
-NdBd4xB8IeSjZDGeMw5wqUXrta0w7dNJjEiYs9/4lh3BkA/x/NPRfjkYmHk6wixj
-YGs6rpolACqRBnZ612qoRI7eOrSy9WC6gU3sajHlWiOPHECBF7jNHU6aeQH4YIig
-UufHALU1gdb4o8qH6h9QibvCKoqdI7LwnD4Pn53SXAfpnNpkgH0SLzdpG++argd+
-33IQeo6DCNazkQ//qdRlrXFsriGNEq8Aip+614hyDkS3N20MZzHrX2sfOz19oZN4
-5UNvkz7gbys/L4MJL1RFapjgofXFKr7wNPAdWE+ZvJ4Irw88pM1LmtDSS/m1tyLn
-hz51vHcw/sBo90DisgNjPZUvO9kHXb1LTVVVmL6EdO0PFp9b4qlKGWUk+IGKejI6
-5ooSlMSjvPAYlmQvy+dcl5uVu+n9M/S5gPIWGiPxrJc73SzOZaHedTSVd2g9yJCz
-smuKKni7og19wgPrAEPvmVS/uObp+rmae2R5/EqgBL1PAIxnPCJbqUJp/865ti2K
-atQeua1yntn3cTFCpO6CkSQXk2LjMtLglZl4AOmf+9UmKi/AsefvSD8t+wARAQAB
-tCVTaGFvZmVuZyBTaGkgPHNoYW9mZW5nc2hpQGFwYWNoZS5vcmc+iQI3BBMBCgAh
-BQJWCky0AhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEKvJ+IVvz0cgVE4P
-/RucXCEPWyG3GY7pGtgBuouBfhHhrWKAkVd58818NwLfOkvTWbGuGNWBqQ1CRPGk
-8ZhaNoVx3T8RqIOOyLx+ynOB6iBhPrVOyMsIxb7jh1BnMhe2aGTZGrrQCChGZk2H
-cdxVbCipO4Pw+YdUtJ4Mc3Xm14TkwAygGqrfJEKXgiAWBlvzrKp6twc69jgSd+To
-/UIrSFdPSlpbEpPiMuGKfhrz8OxPYy6N+/AgwIqGub/8NX9cteLCXRQmL5h7Z+6f
-7Wa4XGlEDfxCJFnWKsUYHwP5QHkJPfESGdbYgI+nrN8rv25rtINfWaiBgAsFSg4D
-KYZBXAelumDmLD/Rpn+hhjpeC5f6lPCqap1atHoqLx/s8YI8149on049pzA5t/zK
-+G+c8bYf8xm6Ky/dBQP15ExoeeIpjC5A5vNlxenaEXXCya50J4+JLC5DTi9xRU7R
-Il3InFjeRmx1hPeyuYSY2irzBFvDhdt7U0wvwERS0lHMkwvB/MPrEwuB/II2tsFQ
-eZvHFhROYh4H8TqE9qICRtcdqNR8BXsZdEJtno4YH//b58130qJSNwcBF/mt+IcZ
-ryVQe3Y87evcF8RWTvWd4KQxe8BGoGQ5o8U230bxJ2bNYl+6O8Xzg+Q4HGpX4DGw
-EUVf4fE3n9O6kH0p9P6RAgnvlsKtsyubxlWz2zNXFuteuQINBFYKTLQBEADbKpjV
-NxaIuUg+wYSLP5erFvHwqDT5OJCj2y4wbwTzVb/1wZL2QwIPzb8lmlJuRsHo1E+j
-bjQE0I1RWiCAdekIpljBHc7fLsYhV0ONMgQZxlh3lpZCIftKmIPX4Eh93bhDBtRi
-CnYcWa/MDEH8rccwZaz0j+2OstIAv7x5/mUC55eutFtH4m5wRkbAjgcZbu6h0MNf
-KbBB4sOfHyx9L1bDIph42Hoh8rp+sMDYNf5Hnxr3Qt4ojANtQgQJzXudI6GzDZHy
-laIle5nJudyPy/TdF/8QXsUT62fbO8W8e4Q8Ljx192RULGoVYlz8WPkXF1vbL8Yb
-mMJyQ9YCgc5jIg/NusMFT0OXHV3YNQ/tZNh57i27pGO4RolRRbrXPrE/npIGBrsR
-8myPu3ussAMSWF2ejhWupKYS943ujNDOhKqC88R5mwHSzDpfytMUMsElwzrioAPV
-EiCCcJ8/Q0u8RAlF9A629iquIA5pp4Ga+9CLHdeGUohmnpIePz71+sXLW+VUOUlM
-+euttiH/ltnRV0LKvS0w6sSnfOFOcOGZKxJ1FJhD5rx86dy7vjrWveOfoCNKjm3a
-jeTOfptvuXzJy0rEgVsKBRLCosjJIVgyWR3UkyXFCq3HfS0IutmdKMzxO+/J6t0c
-pnUsIHEiMlV/2TjwM/icFfUSDzAM/D+OoYe/IQARAQABiQIfBBgBCgAJBQJWCky0
-AhsMAAoJEKvJ+IVvz0cgJ0QQAJt5YtBkMKzPc/QGQo9T86ArCijnt2Z0P6d3S0Qr
-dye87F8ZmM3MRl/Bd7EpZ/+l3157FTX3jBKBrWsKTFcrG5IXrbMTS81pi555AwBm
-HcwCDBdFaLBKlqmQemgNKgnOGt+/G3w1oetPlbkKQEgFjCFfMF8CS8HdNqJU8pmd
-RE3IHMEvwrqx1dbGntRvUrSxs+rRgjJHympIAwIPdLHO6YSIrsbZEghUeVzd/5xi
-cEN06IieGY5t7PImpxeTmYAAdbQsN2GvYOG8NdOY7GrQ2Pu83mreb5Ma/Ui5UtPd
-3KWanXTP4lliY1DYkphp8USjdTul+srOK6X4dQ4y3EYyOzOytvMNV+G4bcYoTnoS
-gUkT1p0oHArxrQej3Py+no4Y5FuYEPJdZ3oHOhkqgn9j/UZBRxfc6VMRAD3ymRXC
-I7eDwdwOryyLBvA16L1+gs2kc5cbbfeFgfFiEFlMfT3eUN9mu5cQ3dNCQ119fcJp
-cu3E67ykSeibobuwNFj+JCilCUGrV5KN3FPVet3TX8F09O9z0CJwjmiK6+JkvVV8
-GB8WCCu0wEwSHYlTZkzOmXMNYZjk8fNYkWplWbF6xNPs1A00XqDwsa/V0F9EzJV+
-nYr5b8EWaM9qsjRh2aFPFtV+E/cSc7sdJxIY/TSdMfqhB13l7LTU/NwmgjE2TTUH
-WYIE
-=6yhP
------END PGP PUBLIC KEY BLOCK-----
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index e2d4f7b..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,247 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-
-
-==============================================================================
-Apache Kylin Subcomponents:
-
-The Apache Kylin project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-
-==============================================================================
-For AdminLTE:
-==============================================================================
-This product bundles AdminLTE, which is available under a
-"MIT" license. For details, see
-https://github.com/almasaeed2010/AdminLTE/blob/master/LICENSE.
-
-
-==============================================================================
-For english-words.80 (scowl-2015.05.18).txt:
-==============================================================================
-This product bundles SCOWL, which is available under a
-MIT-like license. For details, see
-https://raw.githubusercontent.com/kevina/wordlist/master/scowl/Copyright
-
-==============================================================================
-For Google Fonts and Source Sans Pro
-==============================================================================
-This product bundles Google Fonts:
-
-Q: Can I use the Fonts API on any page?
-A: Yes. All the fonts on Google Fonts are licensed under open source licenses
-that let you use them on any website, from a private blog to a big commercial site.
-https://developers.google.com/fonts/faq?hl=en.
-https://www.google.com/fonts/attribution
-
-Source Sans Pro is under SIL Open Font License 1.1,
-For detail, see
-http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=OFL
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index cd0c1ba..0000000
--- a/NOTICE
+++ /dev/null
@@ -1,6 +0,0 @@
-Apache Kylin
-Copyright 2014-2015 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
deleted file mode 100644
index 657212f..0000000
--- a/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-Apache Kylin
-============
-
-> Extreme OLAP Engine for Big Data
-
-Apache Kylin is an open source Distributed Analytics Engine, contributed by eBay Inc., provides SQL interface and multi-dimensional analysis (OLAP) on Hadoop supporting extremely large datasets.
-
-For more details, see the website [http://kylin.apache.org](http://kylin.apache.org).
-
-Documentation
-=============
-Please refer to [http://kylin.apache.org/docs/](http://kylin.apache.org/docs/).
-
-Get Help
-============
-The fastest way to get response from our developers is to send email to our mail list <de...@kylin.apache.org>,
-and remember to subscribe our mail list via <de...@kylin.apache.org>
-
-License
-============
-Please refer to [LICENSE](https://github.com/apache/kylin/blob/master/LICENSE) file.
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/atopcalcite/.settings/org.eclipse.core.resources.prefs b/atopcalcite/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 29abf99..0000000
--- a/atopcalcite/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/atopcalcite/.settings/org.eclipse.jdt.core.prefs b/atopcalcite/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 7c91e8d..0000000
--- a/atopcalcite/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,379 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
-org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
-org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
-org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
-org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
-org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
-org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
-org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
-org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
-org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
-org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
-org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
-org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
-org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
-org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
-org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
-org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
-org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
-org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
-org.eclipse.jdt.core.compiler.problem.unusedImport=warning
-org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
-org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
-org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
-org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
-org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
-org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
-org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
-org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=false
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
-org.eclipse.jdt.core.formatter.comment.format_line_comments=false
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
-org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
-org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
-org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=999
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_on_off_tags=false
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
-org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
-org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/atopcalcite/.settings/org.eclipse.jdt.ui.prefs b/atopcalcite/.settings/org.eclipse.jdt.ui.prefs
deleted file mode 100644
index 681b0b0..0000000
--- a/atopcalcite/.settings/org.eclipse.jdt.ui.prefs
+++ /dev/null
@@ -1,7 +0,0 @@
-eclipse.preferences.version=1
-formatter_profile=_Space Indent & Long Lines
-formatter_settings_version=12
-org.eclipse.jdt.ui.ignorelowercasenames=true
-org.eclipse.jdt.ui.importorder=java;javax;org;com;
-org.eclipse.jdt.ui.ondemandthreshold=99
-org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
deleted file mode 100644
index 03c926c..0000000
--- a/atopcalcite/pom.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>atopcalcite</artifactId>
- <packaging>jar</packaging>
- <name>Kylin:AtopCalcite</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>1.3-SNAPSHOT</version>
- </parent>
-
- <properties>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-avatica</artifactId>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
deleted file mode 100644
index f88157c..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.calcite.sql2rel;
-
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.sql.validate.SqlValidator;
-import org.apache.calcite.tools.RelBuilder;
-
-/*
- * OVERRIDE POINT:
- * - disable the whole RelFieldTrimmer
- */
-
-public class RelFieldTrimmer {
-
- public RelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) {
- }
-
- public RelFieldTrimmer(SqlValidator validator, RelOptCluster cluster, RelFactories.ProjectFactory projectFactory, RelFactories.FilterFactory filterFactory, RelFactories.JoinFactory joinFactory, RelFactories.SemiJoinFactory semiJoinFactory, RelFactories.SortFactory sortFactory, RelFactories.AggregateFactory aggregateFactory, RelFactories.SetOpFactory setOpFactory) {
- }
-
- public RelNode trim(RelNode rootRel) {
- return rootRel;
- }
-
-}