You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by wa...@apache.org on 2016/03/21 12:58:05 UTC
[3/6] kylin git commit: KYLIN-1249 A client library to help automatic
cube
KYLIN-1249 A client library to help automatic cube
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f96b102c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f96b102c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f96b102c
Branch: refs/heads/KYLIN-1122-B1
Commit: f96b102cc479ae87690db5c64ed746ed44ced661
Parents: 771e76b
Author: huanghua@mininglamp.com <hu...@mininglamp.com>
Authored: Wed Feb 24 23:17:56 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Mon Mar 21 19:57:35 2016 +0800
----------------------------------------------------------------------
tools/kylin_client_tool/client/__init__.py | 2 +
.../client/client_interface.py | 226 ++++++++++++++
tools/kylin_client_tool/client/client_parse.py | 96 ++++++
tools/kylin_client_tool/cube_def.csv | 2 +
tools/kylin_client_tool/cube_names.csv | 2 +
tools/kylin_client_tool/jobs/__init__.py | 2 +
tools/kylin_client_tool/jobs/admin.py | 66 ++++
tools/kylin_client_tool/jobs/build.py | 76 +++++
tools/kylin_client_tool/jobs/cube.py | 187 +++++++++++
tools/kylin_client_tool/kylin_client_tool.py | 5 +
tools/kylin_client_tool/models/__init__.py | 2 +
tools/kylin_client_tool/models/cube.py | 309 +++++++++++++++++++
tools/kylin_client_tool/models/dimension.py | 83 +++++
tools/kylin_client_tool/models/hbase.py | 100 ++++++
tools/kylin_client_tool/models/io/__init__.py | 2 +
tools/kylin_client_tool/models/io/base.py | 65 ++++
tools/kylin_client_tool/models/io/cube.py | 90 ++++++
tools/kylin_client_tool/models/io/dimension.py | 55 ++++
tools/kylin_client_tool/models/io/measure.py | 56 ++++
tools/kylin_client_tool/models/io/readers.py | 96 ++++++
tools/kylin_client_tool/models/job.py | 119 +++++++
tools/kylin_client_tool/models/measure.py | 38 +++
tools/kylin_client_tool/models/object.py | 12 +
tools/kylin_client_tool/models/request.py | 187 +++++++++++
tools/kylin_client_tool/models/rowkey.py | 79 +++++
tools/kylin_client_tool/rest/__init__.py | 2 +
tools/kylin_client_tool/rest/apis.py | 102 ++++++
tools/kylin_client_tool/scheduler/__init__.py | 2 +
.../scheduler/workers/__init__.py | 2 +
.../kylin_client_tool/scheduler/workers/cube.py | 122 ++++++++
tools/kylin_client_tool/settings/__init__.py | 2 +
tools/kylin_client_tool/settings/settings.py | 15 +
tools/kylin_client_tool/setup-mac.sh | 20 ++
tools/kylin_client_tool/setup.sh | 20 ++
34 files changed, 2244 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/client/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/client/__init__.py b/tools/kylin_client_tool/client/__init__.py
new file mode 100644
index 0000000..c942174
--- /dev/null
+++ b/tools/kylin_client_tool/client/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Ni Chunen'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/client/client_interface.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/client/client_interface.py b/tools/kylin_client_tool/client/client_interface.py
new file mode 100644
index 0000000..ceb6095
--- /dev/null
+++ b/tools/kylin_client_tool/client/client_interface.py
@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Ni Chunen'
+import time
+import datetime
+
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+
+from scheduler.workers.cube import CubeWorker
+from jobs.cube import CubeJob
+from models.io.readers import CSVReader
+from jobs.build import CubeBuildJob
+
+
+class ClientJob:
+ @staticmethod
+ def build(cube_name_list, endtime=None):
+ run_cube_job_id = '1'
+ check_cube_job_id = '2'
+ scheduler = BackgroundScheduler()
+ CubeWorker.job_instance_dict = {}
+
+ for cube_name in cube_name_list:
+ CubeWorker.job_instance_dict[cube_name] = None
+
+ CubeWorker.scheduler = scheduler
+ CubeWorker.run_cube_job_id = run_cube_job_id
+ CubeWorker.check_cube_job_id = check_cube_job_id
+ # start the run cube job immediately
+ CubeWorker.run_cube_job(endtime)
+
+ scheduler.add_job(CubeWorker.run_cube_job, 'interval', seconds=30, id=run_cube_job_id, args=[endtime])
+ scheduler.add_job(CubeWorker.check_cube_job, 'interval', seconds=30, id=check_cube_job_id)
+ scheduler.start()
+
+ while True:
+ if CubeWorker.all_finished():
+ print "all cube jobs are finished"
+ scheduler.remove_job(check_cube_job_id)
+ scheduler.remove_job(run_cube_job_id)
+ scheduler.shutdown()
+ break
+
+ time.sleep(15)
+
+ @staticmethod
+ def init(cube_name_list):
+ pass
+
+ @staticmethod
+ def build_cube_from_csv(csv_file, database, endtime=None, timed_build=None, crontab_options=None):
+ cube_dic_list = CSVReader.get_cube_desc_list_from_csv(csv_file, database)
+ cube_name_list = []
+
+ for cube_dic in cube_dic_list:
+ cube_name_list.append(cube_dic['cube_desc'].name)
+
+ print "building cubes for", cube_name_list
+
+ if timed_build is not None and crontab_options is not None:
+ scheduler = BlockingScheduler()
+ if timed_build is 'i' and crontab_options is not None:
+ scheduler.add_job(ClientJob.build, 'interval', hours=int(crontab_options),
+ args=[cube_name_list, endtime])
+ try:
+ scheduler.start()
+ except (KeyboardInterrupt, SystemExit):
+ scheduler.shutdown()
+ elif timed_build is 't' and crontab_options is not None:
+ time_list = crontab_options.split(',')
+ if time_list.__len__() == 6:
+ scheduler.add_job(ClientJob.build, 'date',
+ run_date=datetime.datetime(int(time_list[0]), int(time_list[1]),
+ int(time_list[2]), int(time_list[3]),
+ int(time_list[4]), int(time_list[5])),
+ args=[cube_name_list, endtime])
+ try:
+ scheduler.start()
+ except (KeyboardInterrupt, SystemExit):
+ scheduler.shutdown()
+ else:
+ print 'Bad command line!'
+ else:
+ print 'Bad command line!'
+ else:
+ ClientJob.build(cube_name_list, endtime)
+
+ @staticmethod
+ def build_cube_from_names_or_file(cube_name, names_file, endtime=None, timed_build=None, crontab_options=None):
+ cube_name_list_from_file = []
+
+ if names_file is not None:
+ cube_name_list_from_file = CSVReader.get_cube_names_from_csv(names_file)
+ if cube_name is not None:
+ cube_name_list = cube_name.split(',')
+
+ cube_name_list += cube_name_list_from_file
+ print "building cubes for", cube_name_list
+
+ if timed_build is not None and crontab_options is not None:
+ scheduler = BlockingScheduler()
+
+ if timed_build is 'i' and crontab_options is not None:
+ scheduler.add_job(ClientJob.build, 'interval', hours=int(crontab_options),
+ args=[cube_name_list, endtime])
+ try:
+ scheduler.start()
+ except (KeyboardInterrupt, SystemExit):
+ scheduler.shutdown()
+ elif timed_build is 't' and crontab_options is not None:
+ time_list = crontab_options.split(',')
+
+ if time_list.__len__() == 6:
+ scheduler.add_job(ClientJob.build, 'date',
+ run_date=datetime.datetime(int(time_list[0]), int(time_list[1]),
+ int(time_list[2]), int(time_list[3]),
+ int(time_list[4]), int(time_list[5])),
+ args=[cube_name_list, endtime])
+ try:
+ scheduler.start()
+ except (KeyboardInterrupt, SystemExit):
+ scheduler.shutdown()
+ else:
+ print 'Bad command line!'
+ else:
+ print 'Bad command line!'
+ else:
+ ClientJob.build(cube_name_list, endtime)
+
+ @staticmethod
+ def create_cube_from_csv(csv_file, project, database):
+ cube_dic_list = CSVReader.get_cube_desc_list_from_csv(csv_file, database)
+
+ for cube_dic in cube_dic_list:
+ print 'creating cube for', cube_dic['cube_desc'].name, '@project=', project
+ # create cube under project default
+ # print cube_desc.to_json()
+ cube_request_result = CubeJob.create_cube(cube_dic['cube_desc'], cube_dic['model_desc'], project)
+ print 'result=', 'OK' if cube_request_result else 'FAILED'
+
+ @staticmethod
+ def check_job_status(cube_name, names_file, status):
+ cube_name_list = []
+ job_instance_list = []
+ job_status = None
+
+ if status is not None:
+ if status is 'R':
+ job_status = CubeJob.RUNNING_JOB_STATUS
+ elif status is 'F':
+ job_status = CubeJob.FINISHED_JOB_STATUS
+ elif status is 'D':
+ job_status = CubeJob.DISCARDED_JOB_STATUS
+ else:
+ job_status = CubeJob.ERROR_JOB_STATUS
+
+ if cube_name is not None:
+ cube_name_list = cube_name.split(',')
+ elif names_file is not None:
+ cube_name_list = CSVReader.get_cube_names_from_csv(names_file)
+
+ if cube_name_list.__len__() > 0:
+ for cube_name in cube_name_list:
+ job_instance_list += CubeJob.get_cube_job(cube_name, job_status)
+ else:
+ job_instance_list = CubeJob.get_cube_job(None, job_status)
+
+ if job_instance_list.__len__() == 0:
+ print "No job found."
+
+ for job_instance in job_instance_list:
+ print "JOB name " + job_instance.name + " of cube " + job_instance.related_cube + "'s status is " + job_instance.get_status()
+
+ @staticmethod
+ def cancel_job(cube_name, names_file):
+ cube_name_list = []
+ job_instance_list = []
+
+ if cube_name is not None:
+ cube_name_list = cube_name.split(',')
+
+ if names_file is not None:
+ cube_name_list += CSVReader.get_cube_names_from_csv(names_file)
+
+ if cube_name_list.__len__() > 0:
+ for cube_name in cube_name_list:
+ job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.ERROR_JOB_STATUS)
+ job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.RUNNING_JOB_STATUS)
+ else:
+ job_instance_list = CubeJob.get_cube_job(None, CubeJob.ERROR_JOB_STATUS)
+
+ if job_instance_list.__len__() == 0:
+ print "No job found."
+
+ for job_instance in job_instance_list:
+ print "Cancel job " + job_instance.name + "\n"
+ CubeBuildJob.cancel_job(job_instance.uuid)
+
+ @staticmethod
+ def resume_job(cube_name, names_file):
+ cube_name_list = []
+ job_instance_list = []
+
+ if cube_name is not None:
+ cube_name_list = cube_name.split(',')
+
+ if names_file is not None:
+ cube_name_list += CSVReader.get_cube_names_from_csv(names_file)
+
+ if cube_name_list.__len__() > 0:
+ for cube_name in cube_name_list:
+ job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.ERROR_JOB_STATUS)
+ else:
+ job_instance_list = CubeJob.get_cube_job(None, CubeJob.ERROR_JOB_STATUS)
+
+ if job_instance_list.__len__() == 0:
+ print "No error job found."
+
+ for job_instance in job_instance_list:
+ print "Resume job " + job_instance.name + "\n"
+ CubeBuildJob.resume_job(job_instance.uuid)
+
+ @staticmethod
+ def __init__():
+ pass
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/client/client_parse.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/client/client_parse.py b/tools/kylin_client_tool/client/client_parse.py
new file mode 100644
index 0000000..8834bc9
--- /dev/null
+++ b/tools/kylin_client_tool/client/client_parse.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Ni Chunen'
+
+from optparse import OptionParser
+from settings.settings import KYLIN_REST_HOST
+from client.client_interface import ClientJob
+
+
+def menu_parser():
+ print 'Current kylin rest host is ' + KYLIN_REST_HOST + ', if not, please quit and modify it from your setting file.'
+ parser = OptionParser()
+ parser.add_option("-c", "--create_cubes", action="store_true",
+ dest="create_cubes",
+ help="Create cubes with descriptions in the csv file to your project.")
+ parser.add_option("-b", "--build_cubes", action="store_true",
+ dest="build_cubes",
+ help="Build cubes with descriptions in the csv file to your project.")
+ parser.add_option("-s", "--check_job_status", action="store_true",
+ dest="check_job_status",
+ help="Check job status with options.")
+ parser.add_option("-k", "--cancel_job", action="store_true",
+ dest="cancel_job",
+ help="Cancel jobs with options.")
+ parser.add_option("-r", "--resume_job", action="store_true",
+ dest="resume_job",
+ help="Resume jobs with options.")
+
+ parser.add_option("-D", "--database",
+ dest="database",
+ default="default",
+ help="Specify your database,[default=%default].")
+ parser.add_option("-P", "--project",
+ dest="project",
+ default="learn_kylin",
+ help="Specify your project,[default=%default].")
+ parser.add_option("-T", "--time",
+ dest="time",
+ default=None,
+ help="Set the end time of cube building,[default=%default].")
+ parser.add_option("-F", "--cubeDefFile",
+ dest="cubeDefFile",
+ default="cube_def.csv",
+ help="Specify your cubes definition file,[default=%default].")
+ parser.add_option("-f", "--cubeNameFile",
+ dest="cubeNameFile",
+ default=None,
+ help="Specify your cube names' file,[default=%default].")
+ parser.add_option("-S", "--status",
+ dest="status",
+ default=None,
+ help="Specify the job status, R for Running, E for Error, F for Finished, D for Discarded, [default=%default].")
+ parser.add_option("-C", "--cube_name",
+ dest="cube_name",
+ default=None,
+ help="Specify the cube name, [default=%default].")
+ parser.add_option("-B", "--schedule_build",
+ dest="schedule_build",
+ default=None,
+ help="Schedule cube building with options, 'i' for intervally build, 't' for time build, [default=%default].")
+ parser.add_option("-O", "--crontab_options",
+ dest="crontab_options",
+ default=None,
+ help="Set the options of timed building, like '-B i -O 24' for building every 24 hours, '-B t -O 2016,3,1,0,0,0' for building at '2016-3-1 0:0:0', [default=%default].")
+
+ (options, args) = parser.parse_args()
+ status = True
+
+ if options.create_cubes == True and options.cubeDefFile is not None and status == True:
+ ClientJob.create_cube_from_csv(options.cubeDefFile, options.project, options.database)
+ status = False
+
+ if options.build_cubes == True and (
+ options.cubeNameFile is not None or options.cube_name is not None) and status == True:
+ ClientJob.build_cube_from_names_or_file(options.cube_name, options.cubeNameFile, options.time,
+ options.schedule_build, options.crontab_options)
+ status = False
+
+ if options.build_cubes == True and options.cubeDefFile is not None and status == True:
+ ClientJob.build_cube_from_csv(options.cubeDefFile, options.database, options.time, options.schedule_build,
+ options.crontab_options)
+ status = False
+
+ if options.check_job_status == True and status == True:
+ ClientJob.check_job_status(options.cube_name, options.cubeNameFile, options.status)
+ status = False
+
+ if options.cancel_job == True and status == True:
+ ClientJob.cancel_job(options.cube_name, options.cubeNameFile)
+ status = False
+
+ if options.resume_job == True and status == True:
+ ClientJob.resume_job(options.cube_name, options.cubeNameFile)
+ status = False
+
+ if status:
+ print 'Bad command line!'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/cube_def.csv
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/cube_def.csv b/tools/kylin_client_tool/cube_def.csv
new file mode 100644
index 0000000..45f7880
--- /dev/null
+++ b/tools/kylin_client_tool/cube_def.csv
@@ -0,0 +1,2 @@
+ client_tool_test1|kylin_sales|PART_DT,date;TRANS_ID,bigint;LSTG_FORMAT_NAME,string|PRICE,max,decimal|no_dictionary=5/TRANS_ID;mandatory_dimension=PART_DT;partition_date_column=PART_DT;partition_date_start=2010-01-01|
+ client_tool_test2|kylin_sales|PART_DT,date;SELLER_ID,bigint;SLR_SEGMENT_CD,smallint|ITEM_COUNT,sum,bigint|aggregation_group=PART_DT/SELLER_ID,PART_DT/SLR_SEGMENT_CD;partition_date_column=PART_DT;partition_date_start=2010-01-01|
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/cube_names.csv
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/cube_names.csv b/tools/kylin_client_tool/cube_names.csv
new file mode 100644
index 0000000..c22d621
--- /dev/null
+++ b/tools/kylin_client_tool/cube_names.csv
@@ -0,0 +1,2 @@
+client_tool_test1
+client_tool_test2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/jobs/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/jobs/__init__.py b/tools/kylin_client_tool/jobs/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/jobs/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/jobs/admin.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/jobs/admin.py b/tools/kylin_client_tool/jobs/admin.py
new file mode 100644
index 0000000..cc41b7e
--- /dev/null
+++ b/tools/kylin_client_tool/jobs/admin.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from rest.apis import KylinRestApi
+
+
+class AdminJob:
+ @staticmethod
+ def get_env():
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_get('admin/env', '')
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ # elif response is not None and response.json() and "does not exist" in str(response.json()):
+ # status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return response
+
+ @staticmethod
+ def get_conf():
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_get('admin/config', '')
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ # elif response is not None and response.json() and "does not exist" in str(response.json()):
+ # status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return response
+
+ @staticmethod
+ def cleanup_storage():
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_delete('admin/storage', '')
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ # elif response is not None and response.json() and "does not exist" in str(response.json()):
+ # status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return status
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/jobs/build.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/jobs/build.py b/tools/kylin_client_tool/jobs/build.py
new file mode 100644
index 0000000..7a46240
--- /dev/null
+++ b/tools/kylin_client_tool/jobs/build.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from rest.apis import KylinRestApi
+from models.job import JobInstance
+
+
+class CubeBuildJob:
+ @staticmethod
+ def rebuild_cube(cube_name, job_build_request):
+ job_instance = None
+
+ try:
+ Kylin_rest_api = KylinRestApi()
+ response = Kylin_rest_api.http_put('cubes/' + cube_name + '/rebuild', '',
+ payload=job_build_request.to_json())
+
+ if KylinRestApi.is_response_ok(response):
+ job_instance = JobInstance.from_json(response.json())
+ else:
+ print response.json()
+ except Exception, ex:
+ print ex
+
+ return job_instance
+
+ @staticmethod
+ def get_job(job_uuid):
+ job_instance = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_get('jobs/' + job_uuid, '')
+
+ if KylinRestApi.is_response_ok(response):
+ job_instance = JobInstance.from_json(response.json())
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return job_instance
+
+ @staticmethod
+ def cancel_job(job_uuid):
+ job_instance = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('jobs/' + job_uuid + '/cancel', '')
+
+ if KylinRestApi.is_response_ok(response):
+ job_instance = JobInstance.from_json(response.json())
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return job_instance
+
+ @staticmethod
+ def resume_job(job_uuid):
+ job_instance = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('jobs/' + job_uuid + '/resume', '')
+
+ if KylinRestApi.is_response_ok(response):
+ job_instance = JobInstance.from_json(response.json())
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return job_instance
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/jobs/cube.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/jobs/cube.py b/tools/kylin_client_tool/jobs/cube.py
new file mode 100644
index 0000000..4c6f775
--- /dev/null
+++ b/tools/kylin_client_tool/jobs/cube.py
@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.request import CubeRequest, JobListRequest
+from models.cube import CubeInstance
+from models.job import JobInstance
+from rest.apis import KylinRestApi
+
+
+class CubeJob:
+ RUNNING_JOB_STATUS = [0, 1, 2]
+ FINISHED_JOB_STATUS = [4]
+ ERROR_JOB_STATUS = [8]
+ DISCARDED_JOB_STATUS = [16]
+
+ @staticmethod
+ def create_cube(cube_desc, model_desc, project=None):
+ cube_request_result = None
+
+ try:
+ # set last modified time to 0
+ cube_desc.last_modified = 0
+ cube_request = CubeRequest.get_cube_request_from_cube_desc(cube_desc, model_desc, project)
+
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_post('cubes', '', payload=cube_request.to_json())
+
+ if KylinRestApi.is_response_ok(response):
+ cube_request_result = CubeRequest.from_json(response.json())
+ # set result to null if the operation is not successful
+ if not cube_request_result.successful:
+ cube_request_result = None
+ print response.json()
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return cube_request_result
+
+ @staticmethod
+ def update_cube(cube_desc, model_desc, project=None):
+ cube_request_result = None
+
+ try:
+ cube_request = CubeRequest.get_cube_request_from_cube_desc(cube_desc, model_desc, project)
+
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('cubes', '', payload=cube_request.to_json())
+
+ if KylinRestApi.is_response_ok(response):
+ cube_request_result = CubeRequest.from_json(response.json())
+ # set result to null if the operation is not successful
+ if not cube_request_result.successful:
+ cube_request_result = None
+ print response.json()
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return cube_request_result
+
+ @staticmethod
+ def delete_cube(cube_name):
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_delete('cubes/' + cube_name, '')
+ # print response.json()
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ elif response is not None and response.json() and "not found" in str(response.json()):
+ status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return status
+
+ @staticmethod
+ def disable_cube(cube_name):
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('cubes/' + cube_name + '/disable', '')
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ elif response is not None and response.json() and "is DISABLED" in str(response.json()):
+ status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return status
+
+ @staticmethod
+ def enable_cube(cube_name):
+ status = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('cubes/' + cube_name + '/enable', '')
+
+ if KylinRestApi.is_response_ok(response):
+ status = 0
+ elif response is not None and response.json() and "is READY" in str(response.json()):
+ status = 0
+ else:
+ print response.json()
+ # cube_request = None
+ except Exception, ex:
+ pass
+
+ return status
+
+ @staticmethod
+ def get_cube_job(cube_name, status_list=None):
+ job_list_req = JobListRequest()
+ job_list_req.cubeName = cube_name
+ job_list_req.limit = 10000
+ job_list_req.offset = 0
+ job_list_req.status = status_list
+
+ job_instance_list = []
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_get('jobs/', job_list_req.to_query_string())
+
+ if KylinRestApi.is_response_ok(response):
+ job_instance_list = [JobInstance.from_json(json_dict) for json_dict in response.json()]
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return job_instance_list
+
+ @staticmethod
+ def list_cubes(cube_name=None, project_name=None):
+ offset, limit = 0, 10000
+ query_string = '' + ('cubeName=' + cube_name + '&' if cube_name else '') + \
+ ('projectName=' + project_name + '&' if project_name else '') + \
+ ('offset=' + str(offset) + '&') + ('limit=' + str(limit) + '&')
+ cube_instance_list = []
+
+ try:
+ kylin_rest_api = KylinRestApi()
+
+ response = kylin_rest_api.http_get('cubes/', query_string)
+
+ if KylinRestApi.is_response_ok(response):
+ cube_instance_list = [CubeInstance.from_json(json_dict) for json_dict in response.json()]
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return cube_instance_list
+
+ @staticmethod
+ def update_cube_cost(cube_name, cost):
+ cube_instance = None
+
+ try:
+ kylin_rest_api = KylinRestApi()
+ response = kylin_rest_api.http_put('cubes/' + cube_name + '/cost', 'cost=' + str(cost))
+
+ if KylinRestApi.is_response_ok(response):
+ cube_instance = CubeInstance.from_json(response.json())
+ else:
+ print response.json()
+ except Exception, ex:
+ pass
+
+ return cube_instance
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/kylin_client_tool.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/kylin_client_tool.py b/tools/kylin_client_tool/kylin_client_tool.py
new file mode 100644
index 0000000..29d1819
--- /dev/null
+++ b/tools/kylin_client_tool/kylin_client_tool.py
@@ -0,0 +1,5 @@
+__author__ = 'Ni Chunen'
+from client.client_parse import menu_parser
+
+if __name__ == '__main__':
+ menu_parser()
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/__init__.py b/tools/kylin_client_tool/models/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/models/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/cube.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/cube.py b/tools/kylin_client_tool/models/cube.py
new file mode 100644
index 0000000..c2980e6
--- /dev/null
+++ b/tools/kylin_client_tool/models/cube.py
@@ -0,0 +1,309 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.object import JsonSerializableObj
+from models.dimension import DimensionDesc
+from models.measure import MeasureDesc
+from models.rowkey import RowKeyDesc
+from models.hbase import HBaseMappingDesc
+
+
+class CubeDesc(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.cube.model.CubeDesc
+ """
+
+ class CubeDescSetting:
+ APPEND_COUNT_MEASURE = 'append_count_measure'
+ NO_DICTIONARY = 'no_dictionary'
+ MANDATORY_DIMENSION = 'mandatory_dimension'
+ AGGREGATION_GROUP = 'aggregation_group'
+ PARTITION_DATE_COLUMN = 'partition_date_column'
+ PARTITION_DATE_START = 'partition_date_start'
+
+ @staticmethod
+ def get_value(settings, name, default):
+ value = settings.get(name)
+
+ if value is None:
+ return default
+ if value and type(value) == list and value[0].lower() == 'false':
+ return False
+ if value and type(value) == list and value[0].lower() == 'true':
+ return True
+
+ return value
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+ self.name = None
+ self.description = ''
+ self.dimensions = None
+ self.measures = None
+ self.rowkey = None
+ self.notify_list = []
+ self.hbase_mapping = None
+ self.model_name = ''
+ self.retention_range = '0'
+
+ # self.uuid = None
+ # self.last_modified = None
+ # self.fact_table = None
+ # self.null_string = None
+ # self.filter_condition = None
+ # self.cube_partition_desc = None
+ # self.signature = None
+
+ def append_count_measure(self):
+ no_count_measure = True
+ if self.measures:
+ for measure in self.measures:
+ if measure.name == '_COUNT_':
+ no_count_measure = False
+
+ if no_count_measure:
+ if not self.measures: self.measures = []
+
+ self.measures.append(MeasureDesc.get_count_measure(len(self.measures) + 1))
+
+ def apply_settings(self, settings):
+ append_count = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.APPEND_COUNT_MEASURE, True)
+ no_dictionary = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.NO_DICTIONARY, None)
+ mandatory_dimension = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.MANDATORY_DIMENSION,
+ None)
+ aggregation_group = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.AGGREGATION_GROUP,
+ None)
+
+ # apply append_count setting
+ if append_count is None or append_count is True:
+ self.append_count_measure()
+ # update hbase_mapping
+ self.hbase_mapping = HBaseMappingDesc.get_from_measures(self.measures)
+
+ # apply no_dictionary setting
+ if no_dictionary and type(no_dictionary) == list and self.rowkey.rowkey_columns:
+ rowkey_column_cnt = len(self.rowkey.rowkey_columns)
+ for no_dictionary_tuple in no_dictionary:
+ fields = no_dictionary_tuple.split('/')
+ rowkey_size = fields[0]
+ column_name = fields[1]
+ for i in range(rowkey_column_cnt):
+ if self.rowkey.rowkey_columns[i].column == column_name:
+ self.rowkey.rowkey_columns[i].length = int(rowkey_size)
+ self.rowkey.rowkey_columns[i].dictionary = None
+
+ if aggregation_group and type(aggregation_group) == list:
+ agg_groups = []
+ for aggregation_group_tuple in aggregation_group:
+ fields = aggregation_group_tuple.split('/')
+ one_group = []
+ for i in range(len(fields)):
+ one_group.append(fields[i])
+ agg_groups.append(one_group)
+ self.rowkey.aggregation_groups = agg_groups
+
+ # apply mandatory_dimension setting
+ if mandatory_dimension and type(
+ mandatory_dimension) == list and self.rowkey.rowkey_columns and self.rowkey.aggregation_groups:
+ rowkey_column_cnt = len(self.rowkey.rowkey_columns)
+ for m_dim in mandatory_dimension:
+ for i in range(rowkey_column_cnt):
+ if self.rowkey.rowkey_columns[i].column == m_dim:
+ self.rowkey.rowkey_columns[i].mandatory = True
+
+ agg_group_cnt = len(self.rowkey.aggregation_groups)
+ for i in range(agg_group_cnt):
+ new_agg_group = []
+ for dim in self.rowkey.aggregation_groups[i]:
+ if mandatory_dimension.count(dim) <= 0:
+ new_agg_group.append(dim)
+ self.rowkey.aggregation_groups[i] = new_agg_group
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ cd = CubeDesc()
+
+ cd.name = json_dict.get('name')
+ cd.description = json_dict.get('description')
+ if json_dict.get('dimensions') and type(json_dict.get('dimensions')) == list:
+ dimension_list = json_dict.get('dimensions')
+ cd.dimensions = [DimensionDesc.from_json(dimension) for dimension in dimension_list]
+ if json_dict.get('measures') and type(json_dict.get('measures')) == list:
+ measure_list = json_dict.get('measures')
+ cd.measures = [MeasureDesc.from_json(measure) for measure in measure_list]
+ # deserialize json for rowkey
+ cd.rowkey = RowKeyDesc.from_json(json_dict.get('rowkey'))
+ cd.notify_list = json_dict.get('notify_list')
+ # cd.capacity = json_dict.get('capacity')
+ # deserialize json for hbase_mapping
+ cd.hbase_mapping = HBaseMappingDesc.from_json(json_dict.get('hbase_mapping'))
+ cd.retention_range = json_dict.get('retention_range')
+ return cd
+ # cd.uuid = json_dict.get('uuid')
+ # cd.last_modified = json_dict.get('last_modified')
+ # cd.fact_table = json_dict.get('fact_table')
+ # cd.null_string = json_dict.get('null_string')
+ # cd.filter_condition = json_dict.get('filter_condition')
+ # deserialize json for cube_partition_desc
+ # cd.cube_partition_desc = CubePartitionDesc.from_json(json_dict.get('cube_partition_desc'))
+ # deserialize json for dimensions
+
+ # deserialize json for measures
+ # cd.signature = json_dict.get('signature')
+
+
+class CubeModel(JsonSerializableObj):
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+ self.name = None
+ self.fact_table = None
+ self.lookups = []
+ self.filter_condition = ''
+ self.capacity = 'MEDIUM'
+ self.partition_desc = None
+ self.last_modified = 0
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+ cm = CubeModel()
+
+ cm.name = json_dict.get('name')
+ cm.fact_table = json_dict.get('json_dict')
+ if json_dict.get('lookups') and type(json_dict.get('lookups')) == list:
+ lookups_list = json_dict.get('lookups')
+ cm.lookups = lookups_list
+ cm.filter_condition = json_dict.get('json_dict')
+ cm.capacity = json_dict.get('json_dict')
+ cm.partition_desc = CubePartitionDesc.from_json(json_dict.get('partition_desc'))
+ cm.last_modified = json_dict.get('last_modified')
+ return cm
+
+
+class CubePartitionDesc(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.cube.model.v1.CubePartitionDesc
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.partition_date_column = ''
+ self.partition_date_start = ''
+ self.cube_partition_type = 'APPEND'
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ cpd = CubePartitionDesc()
+
+ cpd.partition_date_column = json_dict.get('partition_date_column')
+ cpd.partition_date_start = json_dict.get('partition_date_start')
+ cpd.cube_partition_type = json_dict.get('cube_partition_type')
+
+ return cpd
+
+ @staticmethod
+ def get_default():
+ json_dict = {'partition_date_start': 0, 'cube_partition_type': 'APPEND', 'partition_date_column': None}
+
+ return CubePartitionDesc.from_json(json_dict)
+
+ @staticmethod
+ def get_from_setting(settings):
+ desc = CubePartitionDesc()
+
+ desc.partition_date_start = \
+ CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.PARTITION_DATE_START, [None])[0]
+ desc.cube_partition_type = 'APPEND'
+ desc.partition_date_column = \
+ CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.PARTITION_DATE_COLUMN, [None])[0]
+ return desc
+
+
+class CubeInstance(JsonSerializableObj):
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.name = None
+ self.owner = None
+ self.version = None
+ self.descName = None
+ self.cost = None
+ self.status = None
+ self.segments = None
+ self.create_time = None
+ self.size_kb = None
+ self.source_records_count = None
+ self.source_records_size = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ ci = CubeInstance()
+
+ ci.name = json_dict.get('name')
+ ci.owner = json_dict.get('owner')
+ ci.version = json_dict.get('version')
+ ci.descName = json_dict.get('descName')
+ ci.cost = json_dict.get('cost')
+ ci.status = json_dict.get('status')
+ # deserialize json for segments
+ if json_dict.get('segments') and type(json_dict.get('segments')) == list:
+ segment_list = json_dict.get('segments')
+ ci.segments = [CubeSegment.from_json(segment) for segment in segment_list]
+ ci.create_time = json_dict.get('create_time')
+ ci.size_kb = json_dict.get('size_kb')
+ ci.source_records_count = json_dict.get('source_records_count')
+ ci.source_records_size = json_dict.get('source_records_size')
+
+ return ci
+
+
+class CubeSegment(JsonSerializableObj):
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.uuid = None
+ self.name = None
+ self.storage_location_identifier = None
+ self.date_range_start = None
+ self.date_range_end = None
+ self.status = None
+ self.size_kb = None
+ self.source_records = None
+ self.source_records_size = None
+ self.last_build_time = None
+ self.last_build_job_id = None
+ self.create_time = None
+ self.binary_signature = None
+ self.dictionaries = None
+ self.snapshots = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ cs = CubeSegment()
+
+ cs.uuid = json_dict.get('uuid')
+ cs.name = json_dict.get('name')
+ cs.storage_location_identifier = json_dict.get('storage_location_identifier')
+ cs.date_range_start = json_dict.get('date_range_start')
+ cs.date_range_end = json_dict.get('date_range_end')
+ cs.status = json_dict.get('status')
+ cs.size_kb = json_dict.get('size_kb')
+ cs.source_records = json_dict.get('source_records')
+ cs.source_records_size = json_dict.get('source_records_size')
+ cs.last_build_time = json_dict.get('last_build_time')
+ cs.last_build_job_id = json_dict.get('last_build_job_id')
+ cs.create_time = json_dict.get('create_time')
+ cs.binary_signature = json_dict.get('binary_signature')
+ cs.dictionaries = json_dict.get('dictionaries')
+ cs.snapshots = json_dict.get('snapshots')
+
+ return cs
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/dimension.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/dimension.py b/tools/kylin_client_tool/models/dimension.py
new file mode 100644
index 0000000..0d9446d
--- /dev/null
+++ b/tools/kylin_client_tool/models/dimension.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+
+class DimensionDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.DimensionDesc
+ """
+
+ def __init__(self):
+ self.id = None
+ self.name = None
+ self.join = None
+ self.hierarchy = None
+ self.table = None
+ self.column = []
+ self.datatype = None
+ self.derived = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ dd = DimensionDesc()
+
+ dd.id = json_dict.get('id')
+ dd.name = json_dict.get('name')
+ # deserialize json for join
+ dd.join = JoinDesc.from_json(json_dict.get('join'))
+ # deserialize json for hierarchy
+ if json_dict.get('hierarchy') and type(json_dict.get('hierarchy')) == list:
+ hierarchy_list = json_dict.get('hierarchy')
+ dd.hierarchy = [HierarchyDesc.from_json(hierarchy) for hierarchy in hierarchy_list]
+ dd.table = json_dict.get('table')
+ dd.column = json_dict.get('column')
+ dd.datatype = json_dict.get('datatype')
+ dd.derived = json_dict.get('derived')
+
+ return dd
+
+
+class JoinDesc:
+ """
+ python class mapping to org.apache.kylin.metadata.model.JoinDesc
+ """
+
+ def __init__(self):
+ self.type = None
+ self.primary_key = None
+ self.foreign_key = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ jd = JoinDesc()
+
+ jd.type = json_dict.get('type')
+ jd.primary_key = json_dict.get('primary_key')
+ jd.foreign_key = json_dict.get('foreign_key')
+
+ return jd
+
+
+class HierarchyDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.HierarchyDesc
+ """
+
+ def __init__(self):
+ self.level = None
+ self.column = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ hd = HierarchyDesc()
+
+ hd.level = json_dict.get('level')
+ hd.column = json_dict.get('column')
+
+ return hd
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/hbase.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/hbase.py b/tools/kylin_client_tool/models/hbase.py
new file mode 100644
index 0000000..e8dac1a
--- /dev/null
+++ b/tools/kylin_client_tool/models/hbase.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+
+class HBaseMappingDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.HBaseMappingDesc
+ """
+
+ def __init__(self):
+ self.column_family = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ hmd = HBaseMappingDesc()
+
+ # deserialize json for columns
+ if json_dict.get('column_family') and type(json_dict.get('column_family')) == list:
+ column_family_list = json_dict.get('column_family')
+ hmd.column_family = [HBaseColumnFamilyDesc.from_json(column_family) for column_family in column_family_list]
+
+ return hmd
+
+ @staticmethod
+ def get_from_measures(measures):
+ hbmd = HBaseMappingDesc()
+ hbmd.column_family = []
+
+ if not measures: return hbmd
+
+ start, step, measure_cnt = 0, 5, len(measures)
+ family_id = 1
+ while start < measure_cnt:
+ hbcfd = HBaseColumnFamilyDesc()
+ hbcfd.name = 'F' + str(family_id)
+ hbcfd.columns = []
+
+ hbcd = HBaseColumnDesc()
+ hbcd.qualifier = 'M'
+ hbcd.measure_refs = []
+
+ for measure in measures[start:start + step]:
+ hbcd.measure_refs.append(measure.name)
+
+ # append column instance
+ hbcfd.columns.append(hbcd)
+ # append column family instance
+ hbmd.column_family.append(hbcfd)
+
+ start += step
+ family_id += 1
+
+ return hbmd
+
+
+class HBaseColumnFamilyDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.HBaseColumnFamilyDesc
+ """
+
+ def __init__(self):
+ self.name = None
+ self.columns = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ hcfd = HBaseColumnFamilyDesc()
+
+ hcfd.name = json_dict.get('name')
+ # deserialize json for columns
+ if json_dict.get('columns') and type(json_dict.get('columns')) == list:
+ column_list = json_dict.get('columns')
+ hcfd.columns = [HBaseColumnDesc.from_json(column) for column in column_list]
+
+ return hcfd
+
+
+class HBaseColumnDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.HBaseColumnDesc
+ """
+
+ def __init__(self):
+ self.qualifier = None
+ self.measure_refs = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ hcd = HBaseColumnDesc()
+
+ hcd.qualifier = json_dict.get('qualifier')
+ hcd.measure_refs = json_dict.get('measure_refs')
+
+ return hcd
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/__init__.py b/tools/kylin_client_tool/models/io/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/base.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/base.py b/tools/kylin_client_tool/models/io/base.py
new file mode 100644
index 0000000..cdca483
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/base.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+
+class CSV:
+ def __init__(self, csv_line, sep):
+ self.csv_line = csv_line
+ self.content_list = csv_line.split(sep) if csv_line and isinstance(csv_line, str) else None
+
+ def get_property(self, ind):
+ if self.content_list and len(self.content_list) > ind:
+ return self.content_list[ind]
+ return None
+
+ def is_object_valid(self, cls):
+ if hasattr(cls, 'not_null_attrs'):
+ for attr in getattr(cls, 'not_null_attrs'):
+ if not hasattr(self, attr) or not getattr(self, attr):
+ return False
+ return True
+
+
+class FunctionDesc:
+ """
+ python class mapping to org.apache.kylin.metadata.model.FunctionDesc
+ """
+
+ def __init__(self):
+ self.expression = None
+ self.parameter = None
+ self.returntype = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ fd = FunctionDesc()
+
+ fd.expression = json_dict.get('expression')
+ # deserialize json for parameter
+ fd.parameter = ParameterDesc.from_json(json_dict.get('parameter'))
+ fd.returntype = json_dict.get('returntype')
+
+ return fd
+
+
+class ParameterDesc:
+ """
+ python class mapping to org.apache.kylin.metadata.model.ParameterDesc
+ """
+
+ def __init__(self):
+ self.type = None
+ self.value = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ pd = ParameterDesc()
+
+ pd.type = json_dict.get('type')
+ pd.value = json_dict.get('value')
+
+ return pd
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/cube.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/cube.py b/tools/kylin_client_tool/models/io/cube.py
new file mode 100644
index 0000000..9474855
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/cube.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.io.base import CSV
+from models.io.dimension import DimensionCSV
+from models.io.measure import MeasureCSV
+
+
+class CubeCSV(CSV):
+ columns = {'name': 0, 'table': 1, 'dimensions': 2, 'measures': 3, 'settings': 4, 'filter': 5}
+ not_null_attrs = ['name', 'table', 'dimensions']
+
+ def __init__(self, csv_line, db):
+ CSV.__init__(self, csv_line, '|')
+
+ self.name = self.get_property(CubeCSV.columns['name'])
+ self.table = self.get_property(CubeCSV.columns['table'])
+ self.dimensions = None
+ self.measures = None
+ self.settings = {}
+ self.filter = None
+
+ dimensions_csv = self.get_property(CubeCSV.columns['dimensions'])
+ self.dimensions = DimensionCSV.get_list_from_csv(dimensions_csv, self.table, db)
+
+ measures_csv = self.get_property(CubeCSV.columns['measures'])
+ self.measures = MeasureCSV.get_list_from_csv(measures_csv, 'column')
+ # print measures_csv, len(self.measures) if self.measures else 0
+
+ settings_csv = self.get_property(CubeCSV.columns['settings'])
+ self.settings = CubeSettingCSV.get_settings_from_csv(settings_csv)
+
+ filter_csv = self.get_property(CubeCSV.columns['filter'])
+ self.filter = filter_csv
+
+ def is_valid(self):
+ if not self.is_object_valid(CubeCSV):
+ print self.to_cube_desc_json(), 'not ok'
+ return False
+
+ if self.dimensions:
+ dim_name_dict = {}
+ for dimension in self.dimensions:
+ if not dimension.is_valid() or dimension.name in dim_name_dict:
+ print dimension.to_dimension_desc_json(), 'not ok'
+ return False
+ dim_name_dict[dimension.name] = True
+
+ if self.measures:
+ measure_name_dict = {}
+ for measure in self.measures:
+ if not measure.is_valid() or measure.name in measure_name_dict:
+ print measure.to_measure_desc_json(), 'not ok'
+ return False
+ measure_name_dict[measure.name] = True
+
+ return True
+
+ def to_cube_desc_json(self):
+ json_dict = {'name': self.name, 'fact_table': self.table, 'capacity': 'MEDIUM',
+ 'dimensions': [dimension.to_dimension_desc_json() for dimension in self.dimensions],
+ 'measures': [measure.to_measure_desc_json() for measure in self.measures]}
+
+ return json_dict
+
+
+class CubeSettingCSV(CSV):
+ columns = {'name': 0, 'value': 1}
+
+ def __init__(self, csv_line):
+ CSV.__init__(self, csv_line, '=')
+
+ self.name = self.get_property(CubeSettingCSV.columns['name'])
+ self.value = self.get_property(CubeSettingCSV.columns['value'])
+
+ if self.value:
+ self.value = self.value.split(',')
+
+ @staticmethod
+ def get_settings_from_csv(csv_line):
+ if not csv_line: return {}
+
+ settings = {}
+ setting_csv_list = csv_line.split(';')
+ for csv in setting_csv_list:
+ setting_csv = CubeSettingCSV(csv)
+ if setting_csv:
+ settings[setting_csv.name] = setting_csv.value
+
+ return settings
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/dimension.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/dimension.py b/tools/kylin_client_tool/models/io/dimension.py
new file mode 100644
index 0000000..5276743
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/dimension.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.io.base import CSV
+
+
+class DimensionCSV(CSV):
+ columns = {'column': 0, 'data_type': 1}
+ not_null_attrs = ['id', 'table', 'column', 'name']
+
+ def __init__(self, csv_line, db):
+ CSV.__init__(self, csv_line, ',')
+
+ self.id = None
+ self.table = None
+ self.column = []
+ self.column.append(self.get_property(DimensionCSV.columns['column']))
+ self.data_type = self.get_property(DimensionCSV.columns['data_type'])
+ self.name = self.column[0]
+
+ def to_dimension_desc_json(self):
+ json_dict = {'id': self.id, 'table': self.table, 'datatype': self.data_type,
+ 'column': self.column, 'name': self.name}
+
+ return json_dict
+
+ def is_valid(self):
+ return self.is_object_valid(DimensionCSV)
+
+ @staticmethod
+ def get_from_csv(csv_line, id, table, db):
+ d_csv = None
+
+ if csv_line:
+ d_csv = DimensionCSV(csv_line, db)
+ d_csv.id = id
+ d_csv.table = db + '.' + table
+ d_csv.name = d_csv.table + '.' + d_csv.name
+
+ return d_csv
+
+ @staticmethod
+ def get_list_from_csv(csv_line, table, db):
+ if not csv_line: return []
+
+ dimension_list = []
+ dimension_csv_list = csv_line.split(';')
+ dim_id = 1
+ for csv in dimension_csv_list:
+ dim_csv = DimensionCSV.get_from_csv(csv, dim_id, table, db)
+ if dim_csv:
+ dimension_list.append(dim_csv)
+ dim_id += 1
+
+ return dimension_list
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/measure.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/measure.py b/tools/kylin_client_tool/models/io/measure.py
new file mode 100644
index 0000000..90ed65d
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/measure.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.io.base import CSV, FunctionDesc, ParameterDesc
+
+
+class MeasureCSV(CSV):
+ columns = {'value': 0, 'expression': 1, 'return_type': 2}
+ not_null_attrs = ['id', 'name', 'function']
+
+ def __init__(self, csv_line):
+ CSV.__init__(self, csv_line, ',')
+ parameter = ParameterDesc()
+ parameter.value = self.get_property(MeasureCSV.columns['value'])
+ function = FunctionDesc()
+ function.expression = self.get_property(MeasureCSV.columns['expression'])
+ function.parameter = parameter
+ function.returntype = self.get_property(MeasureCSV.columns['return_type'])
+ self.id = None
+ self.function = function
+ self.name = str(self.function.expression) + '_' + str(self.function.parameter.value)
+
+ def to_measure_desc_json(self):
+ json_dict = {'id': self.id, 'name': self.name, 'dependent_measure_ref': None,
+ 'function': {'returntype': self.function.returntype, 'expression': self.function.expression,
+ 'parameter': {'type': self.function.parameter.type,
+ 'value': self.function.parameter.value}}}
+ return json_dict
+
+ def is_valid(self):
+ return self.is_object_valid(MeasureCSV)
+
+ @staticmethod
+ def get_from_csv(csv_line, id, type):
+ m_csv = MeasureCSV(csv_line)
+
+ m_csv.id = id
+ m_csv.function.parameter.type = type
+
+ return m_csv
+
+ @staticmethod
+ def get_list_from_csv(csv_line, type):
+ if not csv_line: return []
+
+ measure_list = []
+ measure_csv_list = csv_line.split(';')
+ m_id = 1
+
+ for csv in measure_csv_list:
+ m_csv = MeasureCSV.get_from_csv(csv, m_id, type)
+ if m_csv:
+ measure_list.append(m_csv)
+ m_id += 1
+
+ return measure_list
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/io/readers.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/io/readers.py b/tools/kylin_client_tool/models/io/readers.py
new file mode 100644
index 0000000..437671c
--- /dev/null
+++ b/tools/kylin_client_tool/models/io/readers.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+import sys, time, datetime
+from models.io.cube import CubeCSV
+from models.cube import CubeDesc, CubeModel
+from models.cube import CubePartitionDesc
+from models.hbase import HBaseMappingDesc
+from models.rowkey import RowKeyDesc
+
+
+class CSVReader:
+ APPEND_COUNT_MEASURE = 'append_count_measure'
+
+ @staticmethod
+ def get_cube_desc_from_csv_line(csv_line, db):
+ cube_dic = {}
+ cube_csv = CubeCSV(csv_line.strip(), db)
+
+ if not cube_csv.is_valid():
+ raise Exception
+
+ cube_desc = CubeDesc()
+ model_desc = CubeModel()
+ cube_desc.name = cube_csv.name
+ cube_desc.model_name = (cube_csv.name).upper()
+ cube_desc.dimensions = cube_csv.dimensions
+ cube_desc.measures = cube_csv.measures
+ if not cube_desc.rowkey:
+ cube_desc.rowkey = RowKeyDesc.get_from_dimensions(cube_desc.dimensions)
+ if not cube_desc.hbase_mapping:
+ # print cube_desc.measures
+ cube_desc.hbase_mapping = HBaseMappingDesc.get_from_measures(cube_desc.measures)
+
+ model_desc.name = cube_csv.name
+ model_desc.fact_table = (db + '.' + cube_csv.table).upper()
+ if not model_desc.partition_desc:
+ model_desc.partition_desc = CubePartitionDesc.get_from_setting(settings=cube_csv.settings)
+ if not (model_desc.partition_desc.partition_date_column is None):
+ model_desc.partition_desc.partition_date_column \
+ = (db + '.' + cube_csv.table + '.' + model_desc.partition_desc.partition_date_column).upper()
+ timestamp = int(time.mktime(datetime.datetime.strptime(model_desc.partition_desc.partition_date_start,
+ "%Y-%m-%d").timetuple())) - time.timezone
+ model_desc.partition_desc.partition_date_start = timestamp * 1000
+
+ # print cube_desc.rowkey.rowkey_columns[0].dictionary
+
+ # apply settings
+ cube_desc.apply_settings(settings=cube_csv.settings)
+ if cube_csv.filter is None:
+ model_desc.filter_condition = ''
+ else:
+ model_desc.filter_condition = cube_csv.filter
+
+ cube_dic['cube_desc'] = cube_desc
+ cube_dic['model_desc'] = model_desc
+ # print cube_desc.rowkey.rowkey_columns[0].dictionary
+
+ return cube_dic
+
+ @staticmethod
+ def get_cube_desc_list_from_csv(csv_file, db):
+ fd = open(csv_file, 'r')
+ csv_lines = fd.readlines()
+
+ cube_desc_list = []
+
+ for csv_line in csv_lines:
+ if csv_line:
+ try:
+ cube_dic = CSVReader.get_cube_desc_from_csv_line(csv_line, db)
+
+ cube_desc_list.append(cube_dic)
+ except Exception, ex:
+ import traceback
+
+ traceback.print_exc()
+ print "can't parse this csv for cube", csv_line
+ sys.exit(1)
+ pass
+
+ return cube_desc_list
+
+ @staticmethod
+ def get_cube_names_from_csv(csv_file):
+ fd = open(csv_file, 'r')
+ csv_lines = fd.readlines()
+
+ cube_names_list = []
+
+ for csv_line in csv_lines:
+ csv_line = csv_line.strip()
+ if csv_line:
+ cube_names_list.append(csv_line)
+
+ return cube_names_list
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/job.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/job.py b/tools/kylin_client_tool/models/job.py
new file mode 100644
index 0000000..74b59be
--- /dev/null
+++ b/tools/kylin_client_tool/models/job.py
@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.object import JsonSerializableObj
+
+
+class CubeJobStatus:
+ RUNNING = 'RUNNING'
+ ERROR = 'ERROR'
+ FINISHED = 'FINISHED'
+ DISCARD = 'DISCARDED'
+
+
+class JobInstance(JsonSerializableObj):
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.uuid = None
+ self.last_modified = None
+ self.name = None
+ self.type = None
+ self.duration = None
+ self.related_cube = None
+ self.related_segment = None
+ self.exec_start_time = None
+ self.exec_end_time = None
+ self.mr_waiting = None
+ self.steps = None
+ self.submitter = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ ji = JobInstance()
+
+ ji.uuid = json_dict.get('uuid')
+ ji.last_modified = json_dict.get('last_modified')
+ ji.name = json_dict.get('name')
+ ji.type = json_dict.get('type')
+ ji.duration = json_dict.get('duration')
+ ji.related_cube = json_dict.get('related_cube')
+ ji.related_segment = json_dict.get('related_segment')
+ ji.exec_start_time = json_dict.get('exec_start_time')
+ ji.exec_end_time = json_dict.get('exec_end_time')
+ ji.mr_waiting = json_dict.get('mr_waiting')
+ # deserialize json for steps
+ if json_dict.get('steps') and type(json_dict.get('steps')) == list:
+ step_list = json_dict.get('steps')
+ ji.steps = [JobStep.from_json(step) for step in step_list]
+ ji.submitter = json_dict.get('submitter')
+
+ return ji
+
+ def get_status(self):
+ if not self.steps:
+ return CubeJobStatus.ERROR
+
+ for job_step in self.steps:
+ if job_step.step_status in CubeJobStatus.ERROR:
+ return CubeJobStatus.ERROR
+ if job_step.step_status in CubeJobStatus.DISCARD:
+ return CubeJobStatus.DISCARD
+
+ # check the last step
+ job_step = self.steps[-1]
+ if job_step.step_status not in CubeJobStatus.FINISHED:
+ return CubeJobStatus.RUNNING
+
+ return CubeJobStatus.FINISHED
+
+ def get_current_step(self):
+ if not self.steps:
+ return 0
+
+ step_id = 1
+ for job_step in self.steps:
+ if job_step.step_status not in CubeJobStatus.FINISHED:
+ return step_id
+ step_id += 1
+
+ return len(self.steps)
+
+
+class JobStep(JsonSerializableObj):
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.name = None
+ self.sequence_id = None
+ self.exec_cmd = None
+ self.interrupt_cmd = None
+ self.exec_start_time = None
+ self.exec_end_time = None
+ self.exec_wait_time = None
+ self.step_status = None
+ self.cmd_type = None
+ self.info = None
+ self.run_async = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ js = JobStep()
+
+ js.name = json_dict.get('name')
+ js.sequence_id = json_dict.get('sequence_id')
+ js.exec_cmd = json_dict.get('exec_cmd')
+ js.interrupt_cmd = json_dict.get('interrupt_cmd')
+ js.exec_start_time = json_dict.get('exec_start_time')
+ js.exec_end_time = json_dict.get('exec_end_time')
+ js.exec_wait_time = json_dict.get('exec_wait_time')
+ js.step_status = json_dict.get('step_status')
+ js.cmd_type = json_dict.get('cmd_type')
+ js.info = json_dict.get('info')
+ js.run_async = json_dict.get('run_async')
+
+ return js
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/measure.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/measure.py b/tools/kylin_client_tool/models/measure.py
new file mode 100644
index 0000000..b26b623
--- /dev/null
+++ b/tools/kylin_client_tool/models/measure.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+from models.io.base import FunctionDesc
+
+
+class MeasureDesc:
+ """
+ python class mapping to org.apache.kylin.metadata.model.MeasureDesc
+ """
+
+ def __init__(self):
+ self.id = None
+ self.name = None
+ self.function = None
+ self.dependent_measure_ref = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ md = MeasureDesc()
+
+ md.id = json_dict.get('id')
+ md.name = json_dict.get('name')
+ # deserialize json for function
+ md.function = FunctionDesc.from_json(json_dict.get('function'))
+ md.dependent_measure_ref = json_dict.get('dependent_measure_ref')
+
+ return md
+
+ @staticmethod
+ def get_count_measure(id):
+ json_dict = {'id': id, 'name': '_COUNT_',
+ 'function': {'expression': 'COUNT', 'returntype': 'bigint',
+ 'parameter': {'type': 'constant', 'value': 1}},
+ 'dependent_measure_ref': None}
+
+ return MeasureDesc.from_json(json_dict)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/object.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/object.py b/tools/kylin_client_tool/models/object.py
new file mode 100644
index 0000000..d07f89f
--- /dev/null
+++ b/tools/kylin_client_tool/models/object.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+import json
+
+
+class JsonSerializableObj:
+ def __init__(self):
+ pass
+
+ def to_json(self):
+ return json.dumps(self, default=lambda o: o.__dict__)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/request.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/request.py b/tools/kylin_client_tool/models/request.py
new file mode 100644
index 0000000..29cd022
--- /dev/null
+++ b/tools/kylin_client_tool/models/request.py
@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+from models.object import JsonSerializableObj
+from models.cube import CubeDesc, CubeModel
+
+
+class CubeRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.CubeRequest
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.uuid = None
+ self.cubeName = None
+ self.cubeDescData = None
+ self.modelDescData = None
+ self.successful = None
+ self.message = None
+ self.cubeDescName = None
+ self.project = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ cr = CubeRequest()
+
+ cr.uuid = json_dict.get('uuid')
+ cr.cubeName = json_dict.get('cubeName')
+ cr.cubeDescData = json_dict.get('cubeDescData')
+ cr.modelDescData = json_dict.get('modelDescData')
+ cr.successful = json_dict.get('successful')
+ cr.message = json_dict.get('message')
+ cr.cubeDescName = json_dict.get('cubeDescName')
+ cr.project = json_dict.get('project')
+
+ return cr
+
+ @staticmethod
+ def get_cube_request_from_cube_desc(cube_desc, model_desc, project=None):
+ if not cube_desc or not isinstance(cube_desc, CubeDesc): return None
+ if not model_desc or not isinstance(model_desc, CubeModel): return None
+
+ cr = CubeRequest()
+
+ # cr.uuid = cube_desc.uuid
+ cr.cubeDescData = cube_desc.to_json()
+ # print cr.cubeDescData
+ cr.modelDescData = model_desc.to_json()
+ cr.cubeName = cube_desc.name
+
+ cr.project = project
+
+ return cr
+
+
+class JobBuildRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.JobBuildRequest
+ """
+ BUILD = 'BUILD'
+ MERGE = 'MERGE'
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.startTime = None
+ self.endTime = None
+ self.buildType = JobBuildRequest.BUILD
+
+
+class JobListRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.JobListRequest
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.cubeName = None
+ self.projectName = None
+ self.offset = None
+ self.limit = None
+ self.status = None
+
+ def to_query_string(self):
+ qs = ""
+
+ if self.cubeName:
+ qs += "cubeName=" + self.cubeName + "&"
+ if self.projectName:
+ qs += "projectName=" + self.projectName + "&"
+ if self.offset is not None:
+ qs += "offset=" + str(self.offset) + "&"
+ if self.limit is not None:
+ qs += "limit=" + str(self.limit) + "&"
+ if self.status:
+ for status in self.status:
+ qs += "status=" + str(status) + "&"
+
+ return qs[:-1] if qs else ""
+
+
+class ProjectRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.CreateProjectRequest
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.name = None
+ self.description = None
+
+
+class SQLRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.SQLRequest
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.sql = None
+ self.project = None
+ self.offset = None
+ self.limit = None
+ self.acceptPartial = None
+
+
+class PrepareSqlRequest(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.PrepareSqlRequest
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.sql = None
+ self.project = None
+ self.offset = None
+ self.limit = None
+ self.acceptPartial = None
+ self.params = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ psr = PrepareSqlRequest()
+
+ psr.sql = json_dict.get('sql')
+ psr.project = json_dict.get('project')
+ psr.offset = json_dict.get('offset')
+ psr.limit = json_dict.get('limit')
+ psr.acceptPartial = json_dict.get('acceptPartial')
+ if json_dict.get('params') and type(json_dict.get('params')) == list:
+ param_list = json_dict.get('params')
+ psr.params = [StateParam.from_json(param) for param in param_list]
+
+ return psr
+
+
+class StateParam(JsonSerializableObj):
+ """
+ python class mapping to org.apache.kylin.rest.request.PrepareSqlRequest.StateParam
+ """
+
+ def __init__(self):
+ JsonSerializableObj.__init__(self)
+
+ self.className = None
+ self.value = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ sp = StateParam()
+
+ sp.className = json_dict.get('className')
+ sp.value = json_dict.get('value')
+
+ return sp
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/models/rowkey.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/models/rowkey.py b/tools/kylin_client_tool/models/rowkey.py
new file mode 100644
index 0000000..f370864
--- /dev/null
+++ b/tools/kylin_client_tool/models/rowkey.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+
+class RowKeyDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.RowKeyDesc
+ """
+
+ def __init__(self):
+ self.rowkey_columns = None
+ self.aggregation_groups = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ rkd = RowKeyDesc()
+
+ # deserialize json for rowkey_columns
+ if json_dict.get('rowkey_columns') and type(json_dict.get('rowkey_columns')) == list:
+ rowkey_column_list = json_dict.get('rowkey_columns')
+ rkd.rowkey_columns = [RowKeyColDesc.from_json(rowkey_column) for rowkey_column in rowkey_column_list]
+ rkd.aggregation_groups = json_dict.get('aggregation_groups')
+
+ return rkd
+
+ @staticmethod
+ def get_from_dimensions(dimensions):
+ rkd = RowKeyDesc()
+ rkd.rowkey_columns = []
+ rkd.aggregation_groups = []
+
+ if not dimensions: return rkd
+
+ aggregation_group = []
+ for dimension in dimensions:
+ rkd.rowkey_columns.append(RowKeyColDesc.get_from_dimension(dimension))
+ aggregation_group.append(dimension.column[0])
+ rkd.aggregation_groups = [aggregation_group]
+
+ return rkd
+
+
+class RowKeyColDesc:
+ """
+ python class mapping to org.apache.kylin.cube.model.RowKeyColDesc
+ """
+
+ def __init__(self):
+ self.column = None
+ self.length = None
+ self.dictionary = None
+ self.mandatory = None
+
+ @staticmethod
+ def from_json(json_dict):
+ if not json_dict or type(json_dict) != dict: return None
+
+ rkcd = RowKeyColDesc()
+
+ rkcd.column = json_dict.get('column')
+ rkcd.length = json_dict.get('length')
+ rkcd.dictionary = json_dict.get('dictionary')
+ rkcd.mandatory = json_dict.get('mandatory')
+
+ return rkcd
+
+ @staticmethod
+ def get_from_dimension(dimension):
+ rkcd = RowKeyColDesc()
+
+ if dimension:
+ rkcd.column = dimension.column[0]
+ rkcd.length = 0
+ rkcd.mandatory = False
+ rkcd.dictionary = 'true'
+
+ return rkcd
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/rest/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/rest/__init__.py b/tools/kylin_client_tool/rest/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/rest/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/rest/apis.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/rest/apis.py b/tools/kylin_client_tool/rest/apis.py
new file mode 100644
index 0000000..b4d7a63
--- /dev/null
+++ b/tools/kylin_client_tool/rest/apis.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
+
+import json
+import sys
+import requests
+from requests.auth import HTTPBasicAuth
+from settings import settings
+
+
+class KylinRestApi:
+ cooikes = None
+
+ def __init__(self):
+ self.host = settings.KYLIN_REST_HOST
+ self.port = settings.KYLIN_REST_PORT
+ self.user = settings.KYLIN_USER
+ self.password = settings.KYLIN_PASSWORD
+ self.rest_path_prefix = settings.KYLIN_REST_PATH_PREFIX
+
+ if not KylinRestApi.cooikes:
+ KylinRestApi.cooikes = KylinRestApi.login(self)
+
+ if not KylinRestApi.cooikes:
+ print "can't set cookies, exiting..."
+ sys.exit(1)
+
+ @staticmethod
+ def login(kylin_rest_api):
+ if kylin_rest_api.user and kylin_rest_api.password:
+ # auth and get back cookies
+ headers = {}
+ headers['content-type'] = 'application/json'
+ req_response = requests.post(kylin_rest_api.get_api_url('user/authentication', ''), \
+ auth=HTTPBasicAuth(kylin_rest_api.user, kylin_rest_api.password))
+ return req_response.cookies
+
+ return None
+
+ @staticmethod
+ def is_response_ok(response):
+ return str(response.status_code) == '200'
+
+ def get_api_url(self, uri, query_string):
+ return self.host + ':' + str(self.port) + self.rest_path_prefix \
+ + '/' + uri + '?' + query_string
+
+ def http_get(self, uri, query_string, headers=None):
+ api_url = self.get_api_url(uri, query_string)
+
+ headers = headers if headers and type(headers) == dict else {}
+ headers['content-type'] = 'application/json'
+
+ req_response = requests.get(api_url, headers=headers, cookies=KylinRestApi.cooikes)
+
+ return req_response
+
+ def http_post(self, uri, query_string, headers=None, payload=None):
+ api_url = self.get_api_url(uri, query_string)
+
+ headers = headers if headers and type(headers) == dict else {}
+ headers['content-type'] = 'application/json'
+
+ if payload:
+ data = payload if type(payload) == str else json.dumps(payload)
+ req_response = requests.post(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes)
+ else:
+ req_response = requests.post(api_url, headers=headers, cookies=KylinRestApi.cooikes)
+
+ return req_response
+
+ def http_put(self, uri, query_string, headers=None, payload=None):
+ api_url = self.get_api_url(uri, query_string)
+
+ headers = headers if headers and type(headers) == dict else {}
+ headers['content-type'] = 'application/json'
+
+ if payload:
+ data = payload if type(payload) == str else json.dumps(payload)
+ req_response = requests.put(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes)
+ else:
+ req_response = requests.put(api_url, headers=headers, cookies=KylinRestApi.cooikes)
+
+ return req_response
+
+ def http_delete(self, uri, query_string, headers=None, payload=None):
+ api_url = self.get_api_url(uri, query_string)
+
+ headers = headers if headers and type(headers) == dict else {}
+ headers['content-type'] = 'application/json'
+
+ # print payload
+
+ if payload:
+ data = payload if type(payload) == str else json.dumps(payload)
+ req_response = requests.delete(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes)
+ else:
+ req_response = requests.delete(api_url, headers=headers, cookies=KylinRestApi.cooikes)
+
+ # print str(req_response.json())
+
+ return req_response
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/scheduler/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/scheduler/__init__.py b/tools/kylin_client_tool/scheduler/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/scheduler/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f96b102c/tools/kylin_client_tool/scheduler/workers/__init__.py
----------------------------------------------------------------------
diff --git a/tools/kylin_client_tool/scheduler/workers/__init__.py b/tools/kylin_client_tool/scheduler/workers/__init__.py
new file mode 100644
index 0000000..1b249ac
--- /dev/null
+++ b/tools/kylin_client_tool/scheduler/workers/__init__.py
@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__author__ = 'Huang, Hua'