You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:46:56 UTC
[35/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.core.resources.prefs b/dictionary/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..04cfa2c
--- /dev/null
+++ b/dictionary/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,6 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
+encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.jdt.core.prefs b/dictionary/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..a903301
--- /dev/null
+++ b/dictionary/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,379 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
+org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
+org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
+org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
+org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
+org.eclipse.jdt.core.compiler.problem.deadCode=warning
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
+org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
+org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
+org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
+org.eclipse.jdt.core.compiler.problem.nullReference=warning
+org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
+org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
+org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
+org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
+org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
+org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
+org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
+org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
+org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
+org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled
+org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
+org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
+org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
+org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
+org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
+org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
+org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
+org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
+org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
+org.eclipse.jdt.core.compiler.problem.unusedImport=warning
+org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
+org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
+org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
+org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
+org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
+org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
+org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
+org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
+org.eclipse.jdt.core.compiler.source=1.7
+org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
+org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
+org.eclipse.jdt.core.formatter.alignment_for_assignment=0
+org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
+org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
+org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
+org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0
+org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
+org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0
+org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
+org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80
+org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
+org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
+org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16
+org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
+org.eclipse.jdt.core.formatter.blank_lines_after_package=1
+org.eclipse.jdt.core.formatter.blank_lines_before_field=0
+org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
+org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
+org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
+org.eclipse.jdt.core.formatter.blank_lines_before_method=1
+org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
+org.eclipse.jdt.core.formatter.blank_lines_before_package=0
+org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
+org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
+org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
+org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
+org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
+org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
+org.eclipse.jdt.core.formatter.comment.format_block_comments=false
+org.eclipse.jdt.core.formatter.comment.format_header=false
+org.eclipse.jdt.core.formatter.comment.format_html=true
+org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false
+org.eclipse.jdt.core.formatter.comment.format_line_comments=false
+org.eclipse.jdt.core.formatter.comment.format_source_code=true
+org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
+org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
+org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
+org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
+org.eclipse.jdt.core.formatter.comment.line_length=80
+org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true
+org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true
+org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false
+org.eclipse.jdt.core.formatter.compact_else_if=true
+org.eclipse.jdt.core.formatter.continuation_indentation=2
+org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
+org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off
+org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on
+org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
+org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true
+org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
+org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
+org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
+org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
+org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
+org.eclipse.jdt.core.formatter.indent_empty_lines=false
+org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
+org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
+org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
+org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false
+org.eclipse.jdt.core.formatter.indentation.size=4
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
+org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
+org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
+org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
+org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
+org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
+org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
+org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
+org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
+org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
+org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
+org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
+org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
+org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
+org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
+org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
+org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert
+org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
+org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
+org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
+org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert
+org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
+org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
+org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
+org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
+org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert
+org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
+org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
+org.eclipse.jdt.core.formatter.join_lines_in_comments=true
+org.eclipse.jdt.core.formatter.join_wrapped_lines=true
+org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
+org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
+org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
+org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
+org.eclipse.jdt.core.formatter.lineSplit=999
+org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
+org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
+org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
+org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
+org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
+org.eclipse.jdt.core.formatter.tabulation.char=space
+org.eclipse.jdt.core.formatter.tabulation.size=4
+org.eclipse.jdt.core.formatter.use_on_off_tags=false
+org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
+org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
+org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true
+org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/.settings/org.eclipse.jdt.ui.prefs
----------------------------------------------------------------------
diff --git a/dictionary/.settings/org.eclipse.jdt.ui.prefs b/dictionary/.settings/org.eclipse.jdt.ui.prefs
new file mode 100644
index 0000000..dece0e6
--- /dev/null
+++ b/dictionary/.settings/org.eclipse.jdt.ui.prefs
@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+formatter_profile=_Space Indent & Long Lines
+formatter_settings_version=12
+org.eclipse.jdt.ui.ignorelowercasenames=true
+org.eclipse.jdt.ui.importorder=java;javax;org;com;
+org.eclipse.jdt.ui.ondemandthreshold=99
+org.eclipse.jdt.ui.staticondemandthreshold=1
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
new file mode 100644
index 0000000..1d09903
--- /dev/null
+++ b/dictionary/pom.xml
@@ -0,0 +1,78 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-dictionary</artifactId>
+ <packaging>jar</packaging>
+ <name>Kylin:Dictionary</name>
+
+ <parent>
+ <groupId>com.kylinolap</groupId>
+ <artifactId>kylin</artifactId>
+ <version>0.6.3-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <!--Kylin Jar -->
+ <dependency>
+ <groupId>com.kylinolap</groupId>
+ <artifactId>kylin-metadata</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <!-- Env & Test -->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ <!-- protobuf version conflict with hbase-->
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java b/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java
new file mode 100644
index 0000000..cff65c5
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.dict;
+
+public interface BytesConverter<T> {
+
+ public byte[] convertToBytes(T v);
+
+ public T convertFromBytes(byte[] b, int offset, int length);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java
new file mode 100644
index 0000000..ed26aae
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.dict;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * A dictionary for date string (date only, no time).
+ *
+ * Dates are numbered from 1970-1-1 -- 0 for 1970-1-1, 1 for 1-2, 2 for 1-3 and
+ * so on. With 2 bytes, 65536 states, can express dates up to the year of 2149.
+ *
+ * Note the implementation is not thread-safe.
+ *
+ * @author yangli9
+ */
+public class DateStrDictionary extends Dictionary<String> {
+
+ static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
+ static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
+ static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
+
+ static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
+
+ static SimpleDateFormat getDateFormat(String datePattern) {
+ ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
+ if (formatThreadLocal == null) {
+ threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
+ }
+ SimpleDateFormat format = formatThreadLocal.get();
+ if (format == null) {
+ format = new SimpleDateFormat(datePattern);
+ format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must
+ // be GMT to
+ // calculate
+ // epoch date
+ // correctly
+ formatThreadLocal.set(format);
+ }
+ return format;
+ }
+
+ public static String dateToString(Date date) {
+ return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ }
+
+ public static String dateToString(Date date, String pattern) {
+ return getDateFormat(pattern).format(date);
+ }
+
+ public static Date stringToDate(String str) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN);
+ }
+
+ public static Date stringToDate(String str, String pattern) {
+ Date date = null;
+ try {
+ date = getDateFormat(pattern).parse(str);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
+ }
+ return date;
+ }
+
+ public static long stringToMillis(String str) {
+ if (str.length() == 10) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
+ } else if (str.length() == 19) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
+ } else if (str.length() == 23) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
+ } else {
+ throw new IllegalArgumentException("there is no valid date pattern for:" + str);
+ }
+ }
+
+ // ============================================================================
+
+ private String pattern;
+ private int baseId;
+
+ public DateStrDictionary() {
+ init(DEFAULT_DATE_PATTERN, 0);
+ }
+
+ public DateStrDictionary(String datePattern, int baseId) {
+ init(datePattern, baseId);
+ }
+
+ private void init(String datePattern, int baseId) {
+ this.pattern = datePattern;
+ this.baseId = baseId;
+ }
+
+ @Override
+ public int getMinId() {
+ return baseId;
+ }
+
+ @Override
+ public int getMaxId() {
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int getSizeOfId() {
+ return 3;
+ }
+
+ @Override
+ public int getSizeOfValue() {
+ return pattern.length();
+ }
+
+ @Override
+ protected boolean isNullByteForm(byte[] value, int offset, int len) {
+ return value == null || len == 0;
+ }
+
+ @Override
+ final protected int getIdFromValueImpl(String value, int roundFlag) {
+ Date date = stringToDate(value, pattern);
+ int id = calcIdFromSeqNo(getNumOfDaysSince0000(date));
+ if (id < 0 || id >= 16777216)
+ throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range of 3 bytes");
+
+ return id;
+ }
+
+ @Override
+ final protected String getValueFromIdImpl(int id) {
+ if (id < baseId)
+ throw new IllegalArgumentException("ID '" + id + "' must not be less than base ID " + baseId);
+ Date d = getDateFromNumOfDaysSince0000(calcSeqNoFromId(id));
+ return dateToString(d, pattern);
+ }
+
+ private int getNumOfDaysSince0000(Date d) {
+ // 86400000 = 1000 * 60 * 60 * 24
+ // -719530 is offset of 0000-01-01
+ return (int) (d.getTime() / 86400000 + 719530);
+ }
+
+ private Date getDateFromNumOfDaysSince0000(int n) {
+ long millis = ((long) n - 719530) * 86400000;
+ return new Date(millis);
+ }
+
+ @Override
+ final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ try {
+ return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ @Override
+ final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ String date = getValueFromId(id);
+ byte bytes[];
+ try {
+ bytes = date.getBytes("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
+ return bytes.length;
+ }
+
+ private int calcIdFromSeqNo(int seq) {
+ return seq < 0 ? seq : baseId + seq;
+ }
+
+ private int calcSeqNoFromId(int id) {
+ return id - baseId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(pattern);
+ out.writeInt(baseId);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ String pattern = in.readUTF();
+ int baseId = in.readInt();
+ init(pattern, baseId);
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * baseId + pattern.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if ((o instanceof DateStrDictionary) == false)
+ return false;
+ DateStrDictionary that = (DateStrDictionary) o;
+ return StringUtils.equals(this.pattern, that.pattern) && this.baseId == that.baseId;
+ }
+
+ @Override
+ public void dump(PrintStream out) {
+ out.println(this.toString());
+ }
+
+ @Override
+ public String toString() {
+ return "DateStrDictionary [pattern=" + pattern + ", baseId=" + baseId + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
new file mode 100644
index 0000000..b54c6a3
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.dict;
+
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.hadoop.io.Writable;
+
+import com.kylinolap.common.util.BytesUtil;
+
+/**
+ * A bi-way dictionary that maps from dimension/column values to IDs and vice
+ * versa. By storing IDs instead of real values, the size of cube is
+ * significantly reduced.
+ *
+ * - IDs are smallest integers possible for the cardinality of a column, for the
+ * purpose of minimal storage space - IDs preserve ordering of values, such that
+ * range query can be applied to IDs directly
+ *
+ * A dictionary once built, is immutable. This allows optimal memory footprint
+ * by e.g. flatten the Trie structure into a byte array, replacing node pointers
+ * with array offsets.
+ *
+ * @author yangli9
+ */
+abstract public class Dictionary<T> implements Writable {
+
+ public static final byte NULL = (byte) 0xff;
+
+ // ID with all bit-1 (0xff e.g.) reserved for NULL value
+ public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff };
+
+ abstract public int getMinId();
+
+ abstract public int getMaxId();
+
+ /**
+ * @return the size of an ID in bytes, determined by the cardinality of
+ * column
+ */
+ abstract public int getSizeOfId();
+
+ /**
+ * @return the (maximum) size of value in bytes, determined by the longest
+ * value of column
+ */
+ abstract public int getSizeOfValue();
+
+ /**
+ * Convenient form of <code>getIdFromValue(value, 0)</code>
+ */
+ final public int getIdFromValue(T value) {
+ return getIdFromValue(value, 0);
+ }
+
+ /**
+ * Returns the ID integer of given value. In case of not found - if
+ * roundingFlag=0, throw IllegalArgumentException; - if roundingFlag<0, the
+ * closest smaller ID integer if exist; - if roundingFlag>0, the closest
+ * bigger ID integer if exist. The implementation often has cache, thus
+ * faster than the byte[] version getIdFromValueBytes()
+ *
+ * @throws IllegalArgumentException
+ * if value is not found in dictionary and rounding is off or
+ * failed
+ */
+ final public int getIdFromValue(T value, int roundingFlag) {
+ if (isNullObjectForm(value))
+ return nullId();
+ else
+ return getIdFromValueImpl(value, roundingFlag);
+ }
+
+ protected boolean isNullObjectForm(T value) {
+ return value == null;
+ }
+
+ abstract protected int getIdFromValueImpl(T value, int roundingFlag);
+
+ /**
+ * @return the value corresponds to the given ID
+ * @throws IllegalArgumentException
+ * if ID is not found in dictionary
+ */
+ final public T getValueFromId(int id) {
+ if (isNullId(id))
+ return null;
+ else
+ return getValueFromIdImpl(id);
+ }
+
+ abstract protected T getValueFromIdImpl(int id);
+
+ /**
+ * Convenient form of
+ * <code>getIdFromValueBytes(value, offset, len, 0)</code>
+ */
+ final public int getIdFromValueBytes(byte[] value, int offset, int len) {
+ return getIdFromValueBytes(value, offset, len, 0);
+ }
+
+ /**
+ * A lower level API, return ID integer from raw value bytes. In case of not
+ * found - if roundingFlag=0, throw IllegalArgumentException; - if
+ * roundingFlag<0, the closest smaller ID integer if exist; - if
+ * roundingFlag>0, the closest bigger ID integer if exist. Bypassing the
+ * cache layer, this could be significantly slower than getIdFromValue(T
+ * value).
+ *
+ * @throws IllegalArgumentException
+ * if value is not found in dictionary and rounding is off or
+ * failed
+ */
+ final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) {
+ if (isNullByteForm(value, offset, len))
+ return nullId();
+ else
+ return getIdFromValueBytesImpl(value, offset, len, roundingFlag);
+ }
+
+ protected boolean isNullByteForm(byte[] value, int offset, int len) {
+ return value == null;
+ }
+
+ abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
+
+ /**
+ * A lower level API, get byte values from ID, return the number of bytes
+ * written. Bypassing the cache layer, this could be significantly slower
+ * than getIdFromValue(T value).
+ *
+ * @throws IllegalArgumentException
+ * if ID is not found in dictionary
+ */
+ final public int getValueBytesFromId(int id, byte[] returnValue, int offset) {
+ if (isNullId(id))
+ return 0;
+ else
+ return getValueBytesFromIdImpl(id, returnValue, offset);
+ }
+
+ abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
+
+ abstract public void dump(PrintStream out);
+
+ public int nullId() {
+ return NULL_ID[getSizeOfId()];
+ }
+
+ public boolean isNullId(int id) {
+ int nullId = NULL_ID[getSizeOfId()];
+ return (nullId & id) == nullId;
+ }
+
+ /** utility that converts a dictionary ID to string, preserving order */
+ public static String dictIdToString(byte[] idBytes, int offset, int length) {
+ try {
+ return new String(idBytes, offset, length, "ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ // never happen
+ return null;
+ }
+ }
+
+ /** the reverse of dictIdToString(), returns integer ID */
+ public static int stringToDictId(String str) {
+ try {
+ byte[] bytes = str.getBytes("ISO-8859-1");
+ return BytesUtil.readUnsigned(bytes, 0, bytes.length);
+ } catch (UnsupportedEncodingException e) {
+ // never happen
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
new file mode 100644
index 0000000..16ae61c
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.dict;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.util.JsonUtil;
+import com.kylinolap.dict.lookup.ReadableTable;
+import com.kylinolap.dict.lookup.TableReader;
+import com.kylinolap.metadata.model.schema.DataType;
+
+/**
+ * @author yangli9
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class DictionaryGenerator {
+
+ private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
+
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd" };
+
+ public static Dictionary<?> buildDictionaryFromValueList(DictionaryInfo info, List<byte[]> values) {
+ info.setCardinality(values.size());
+
+ Dictionary dict = null;
+ int baseId = 0; // always 0 for now
+ int nSamples = 5;
+ ArrayList samples = new ArrayList();
+
+ // build dict, case by data type
+ DataType dataType = DataType.getInstance(info.getDataType());
+ if (dataType.isDateTimeFamily())
+ dict = buildDateStrDict(values, baseId, nSamples, samples);
+ else if (dataType.isNumberFamily())
+ dict = buildNumberDict(values, baseId, nSamples, samples);
+ else
+ dict = buildStringDict(values, baseId, nSamples, samples);
+
+ // log a few samples
+ StringBuilder buf = new StringBuilder();
+ for (Object s : samples) {
+ if (buf.length() > 0)
+ buf.append(", ");
+ buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
+ }
+ logger.info("Dictionary value samples: " + buf.toString());
+ logger.info("Dictionary cardinality " + info.getCardinality());
+
+ if (values.size() > 1000000)
+ throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary! Are the values stable enough for incremental load??");
+
+ return dict;
+ }
+
+ public static Dictionary mergeDictionaries(DictionaryInfo targetInfo, List<DictionaryInfo> sourceDicts) {
+
+ HashSet<byte[]> dedup = new HashSet<byte[]>();
+
+ for (DictionaryInfo info : sourceDicts) {
+ Dictionary<?> dict = info.getDictionaryObject();
+ int minkey = dict.getMinId();
+ int maxkey = dict.getMaxId();
+ byte[] buffer = new byte[dict.getSizeOfValue()];
+ for (int i = minkey; i <= maxkey; ++i) {
+ int size = dict.getValueBytesFromId(i, buffer, 0);
+ dedup.add(Bytes.copy(buffer, 0, size));
+ }
+ }
+
+ List<byte[]> valueList = new ArrayList<byte[]>();
+ valueList.addAll(dedup);
+
+ return buildDictionaryFromValueList(targetInfo, valueList);
+ }
+
+ public static Dictionary<?> buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException {
+
+ // currently all data types are casted to string to build dictionary
+ // String dataType = info.getDataType();
+
+ logger.info("Building dictionary " + JsonUtil.writeValueAsString(info));
+
+ ArrayList<byte[]> values = loadColumnValues(inpTable, info.getSourceColumnIndex());
+
+ return buildDictionaryFromValueList(info, values);
+ }
+
+ private static Dictionary buildDateStrDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ final int BAD_THRESHOLD = 2;
+ String matchPattern = null;
+
+ for (String ptn : DATE_PATTERNS) {
+ matchPattern = ptn; // be optimistic
+ int badCount = 0;
+ SimpleDateFormat sdf = new SimpleDateFormat(ptn);
+ for (byte[] value : values) {
+ if (value.length == 0)
+ continue;
+
+ String str = Bytes.toString(value);
+ try {
+ sdf.parse(str);
+ if (samples.size() < nSamples && samples.contains(str) == false)
+ samples.add(str);
+ } catch (ParseException e) {
+ logger.info("Unrecognized datetime value: " + str);
+ badCount++;
+ if (badCount > BAD_THRESHOLD) {
+ matchPattern = null;
+ break;
+ }
+ }
+ }
+ if (matchPattern != null)
+ return new DateStrDictionary(matchPattern, baseId);
+ }
+ throw new IllegalStateException("Unrecognized datetime value");
+ }
+
+ private static Dictionary buildStringDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter());
+ for (byte[] value : values) {
+ String v = Bytes.toString(value);
+ builder.addValue(v);
+ if (samples.size() < nSamples && samples.contains(v) == false)
+ samples.add(v);
+ }
+ return builder.build(baseId);
+ }
+
+ private static Dictionary buildNumberDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ for (byte[] value : values) {
+ String v = Bytes.toString(value);
+ builder.addValue(v);
+ if (samples.size() < nSamples && samples.contains(v) == false)
+ samples.add(v);
+ }
+ return builder.build(baseId);
+ }
+
+ static ArrayList<byte[]> loadColumnValues(ReadableTable inpTable, int colIndex) throws IOException {
+
+ TableReader reader = inpTable.getReader();
+
+ try {
+ ArrayList<byte[]> result = Lists.newArrayList();
+ HashSet<String> dedup = new HashSet<String>();
+
+ while (reader.next()) {
+ String[] split = reader.getRow();
+
+ String colValue;
+ // special single column file, e.g. common_indicator.txt
+ if (split.length == 1) {
+ colValue = split[0];
+ }
+ // normal case
+ else {
+ if (split.length <= colIndex) {
+ throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split));
+ }
+ colValue = split[colIndex];
+ }
+
+ if (dedup.contains(colValue) == false) {
+ dedup.add(colValue);
+ result.add(Bytes.toBytes(colValue));
+ }
+ }
+ return result;
+
+ } finally {
+ reader.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java
new file mode 100644
index 0000000..89e01b2
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.dict;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.kylinolap.common.persistence.ResourceStore;
+import com.kylinolap.common.persistence.RootPersistentEntity;
+import com.kylinolap.dict.lookup.TableSignature;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class DictionaryInfo extends RootPersistentEntity {
+
+ @JsonProperty("source_table")
+ private String sourceTable;
+ @JsonProperty("source_column")
+ private String sourceColumn;
+ @JsonProperty("source_column_index")
+ private int sourceColumnIndex; // 0 based
+ @JsonProperty("data_type")
+ private String dataType;
+ @JsonProperty("input")
+ private TableSignature input;
+ @JsonProperty("input_delimeter")
+ private String inputDelimeter;
+ @JsonProperty("dictionary_class")
+ private String dictionaryClass;
+ @JsonProperty("cardinality")
+ private int cardinality;
+
+ transient Dictionary<?> dictionaryObject;
+
+ public DictionaryInfo() {
+ }
+
+ public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input, String inputDelimeter) {
+
+ this.updateRandomUuid();
+
+ this.sourceTable = sourceTable;
+ this.sourceColumn = sourceColumn;
+ this.sourceColumnIndex = sourceColumnIndex;
+ this.dataType = dataType;
+ this.input = input;
+ this.inputDelimeter = inputDelimeter;
+ }
+
+ public DictionaryInfo(DictionaryInfo other) {
+
+ this.updateRandomUuid();
+
+ this.sourceTable = other.sourceTable;
+ this.sourceColumn = other.sourceColumn;
+ this.sourceColumnIndex = other.sourceColumnIndex;
+ this.dataType = other.dataType;
+ this.input = other.input;
+ this.inputDelimeter = other.inputDelimeter;
+ }
+
+ // ----------------------------------------------------------------------------
+
+ public String getResourcePath() {
+ return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn + "/" + uuid + ".dict";
+ }
+
+ public String getResourceDir() {
+ return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn;
+ }
+
+ // ----------------------------------------------------------------------------
+
+ // to decide if two dictionaries are built on the same table/column,
+ // regardless of their signature
+ public boolean isDictOnSameColumn(DictionaryInfo other) {
+ return this.sourceTable.equalsIgnoreCase(other.sourceTable) &&
+ this.sourceColumn.equalsIgnoreCase(other.sourceColumn) &&
+ this.sourceColumnIndex == other.sourceColumnIndex &&
+ this.dataType.equalsIgnoreCase(other.dataType) &&
+ this.inputDelimeter.equalsIgnoreCase(other.inputDelimeter) &&
+ this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass);
+ }
+
+ public String getSourceTable() {
+ return sourceTable;
+ }
+
+ public void setSourceTable(String sourceTable) {
+ this.sourceTable = sourceTable;
+ }
+
+ public String getSourceColumn() {
+ return sourceColumn;
+ }
+
+ public void setSourceColumn(String sourceColumn) {
+ this.sourceColumn = sourceColumn;
+ }
+
+ public int getSourceColumnIndex() {
+ return sourceColumnIndex;
+ }
+
+ public void setSourceColumnIndex(int sourceColumnIndex) {
+ this.sourceColumnIndex = sourceColumnIndex;
+ }
+
+ public String getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(String dataType) {
+ this.dataType = dataType;
+ }
+
+ public TableSignature getInput() {
+ return input;
+ }
+
+ public void setInput(TableSignature input) {
+ this.input = input;
+ }
+
+ public String getInputDelimeter() {
+ return inputDelimeter;
+ }
+
+ public void setInputDelimeter(String inputDelimeter) {
+ this.inputDelimeter = inputDelimeter;
+ }
+
+ public String getDictionaryClass() {
+ return dictionaryClass;
+ }
+
+ public void setDictionaryClass(String dictionaryClass) {
+ this.dictionaryClass = dictionaryClass;
+ }
+
+ public Dictionary<?> getDictionaryObject() {
+ return dictionaryObject;
+ }
+
+ public void setDictionaryObject(Dictionary<?> dictionaryObject) {
+ this.dictionaryObject = dictionaryObject;
+ }
+
+ public int getCardinality() {
+ return cardinality;
+ }
+
+ public void setCardinality(int cardinality) {
+ this.cardinality = cardinality;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java
new file mode 100644
index 0000000..623d25a
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.dict;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import com.kylinolap.common.persistence.Serializer;
+import com.kylinolap.common.util.JsonUtil;
+
+/**
+ * @author yangli9
+ *
+ */
+public class DictionaryInfoSerializer implements Serializer<DictionaryInfo> {
+
+ public static final DictionaryInfoSerializer FULL_SERIALIZER = new DictionaryInfoSerializer(false);
+ public static final DictionaryInfoSerializer INFO_SERIALIZER = new DictionaryInfoSerializer(true);
+
+ private boolean infoOnly;
+
+ public DictionaryInfoSerializer() {
+ this(false);
+ }
+
+ public DictionaryInfoSerializer(boolean infoOnly) {
+ this.infoOnly = infoOnly;
+ }
+
+ @Override
+ public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException {
+ String json = JsonUtil.writeValueAsIndentString(obj);
+ out.writeUTF(json);
+
+ if (infoOnly == false)
+ obj.getDictionaryObject().write(out);
+ }
+
+ @Override
+ public DictionaryInfo deserialize(DataInputStream in) throws IOException {
+ String json = in.readUTF();
+ DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class);
+
+ if (infoOnly == false) {
+ Dictionary<?> dict;
+ try {
+ dict = (Dictionary<?>) Class.forName(obj.getDictionaryClass()).newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ dict.readFields(in);
+ obj.setDictionaryObject(dict);
+ }
+ return obj;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java
new file mode 100644
index 0000000..73bc9d9
--- /dev/null
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java
@@ -0,0 +1,345 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.dict;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.persistence.ResourceStore;
+import com.kylinolap.common.util.HadoopUtil;
+import com.kylinolap.dict.lookup.FileTable;
+import com.kylinolap.dict.lookup.HiveTable;
+import com.kylinolap.dict.lookup.ReadableTable;
+import com.kylinolap.dict.lookup.TableSignature;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+import com.kylinolap.metadata.model.cube.TblColRef;
+
+public class DictionaryManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(DictionaryManager.class);
+
+ private static final DictionaryInfo NONE_INDICATOR = new DictionaryInfo();
+
+ // static cached instances
+ private static final ConcurrentHashMap<KylinConfig, DictionaryManager> SERVICE_CACHE = new ConcurrentHashMap<KylinConfig, DictionaryManager>();
+
+ public static DictionaryManager getInstance(KylinConfig config) {
+ DictionaryManager r = SERVICE_CACHE.get(config);
+ if (r == null) {
+ r = new DictionaryManager(config);
+ SERVICE_CACHE.put(config, r);
+ if (SERVICE_CACHE.size() > 1) {
+ logger.warn("More than one singleton exist");
+ }
+ }
+ return r;
+ }
+
+ public static void removeInstance(KylinConfig config) {
+ SERVICE_CACHE.remove(config);
+ }
+
+ // ============================================================================
+
+ private KylinConfig config;
+ private ConcurrentHashMap<String, DictionaryInfo> dictCache; // resource
+ // path ==>
+ // DictionaryInfo
+
+ private DictionaryManager(KylinConfig config) {
+ this.config = config;
+ dictCache = new ConcurrentHashMap<String, DictionaryInfo>();
+ }
+
+ public Dictionary<?> getDictionary(String resourcePath) throws IOException {
+ DictionaryInfo dictInfo = getDictionaryInfo(resourcePath);
+ return dictInfo == null ? null : dictInfo.getDictionaryObject();
+ }
+
+ public DictionaryInfo getDictionaryInfo(String resourcePath) throws IOException {
+ DictionaryInfo dictInfo = dictCache.get(resourcePath);
+ if (dictInfo == null) {
+ dictInfo = load(resourcePath, true);
+ if (dictInfo == null)
+ dictInfo = NONE_INDICATOR;
+ dictCache.put(resourcePath, dictInfo);
+ }
+ return dictInfo == NONE_INDICATOR ? null : dictInfo;
+ }
+
+ public DictionaryInfo trySaveNewDict(Dictionary<?> newDict, DictionaryInfo newDictInfo) throws IOException {
+
+ String dupDict = checkDupByContent(newDictInfo, newDict);
+ if (dupDict != null) {
+ logger.info("Identical dictionary content " + newDict + ", reuse existing dictionary at " + dupDict);
+ return getDictionaryInfo(dupDict);
+ }
+
+ newDictInfo.setDictionaryObject(newDict);
+ newDictInfo.setDictionaryClass(newDict.getClass().getName());
+
+ save(newDictInfo);
+ dictCache.put(newDictInfo.getResourcePath(), newDictInfo);
+
+ return newDictInfo;
+ }
+
+ public DictionaryInfo mergeDictionary(List<DictionaryInfo> dicts) throws IOException {
+ DictionaryInfo firstDictInfo = null;
+ int totalSize = 0;
+ for (DictionaryInfo info : dicts) {
+ // check
+ if (firstDictInfo == null) {
+ firstDictInfo = info;
+ } else {
+ if (!firstDictInfo.isDictOnSameColumn(info)) {
+ throw new IllegalArgumentException("Merging dictionaries are not structurally equal(regardless of signature).");
+ }
+ }
+ totalSize += info.getInput().getSize();
+ }
+
+ if (firstDictInfo == null) {
+ throw new IllegalArgumentException("DictionaryManager.mergeDictionary input cannot be null");
+ }
+
+ DictionaryInfo newDictInfo = new DictionaryInfo(firstDictInfo);
+ TableSignature signature = newDictInfo.getInput();
+ signature.setSize(totalSize);
+ signature.setLastModifiedTime(System.currentTimeMillis());
+ signature.setPath("merged_with_no_original_path");
+
+ String dupDict = checkDupByInfo(newDictInfo);
+ if (dupDict != null) {
+ logger.info("Identical dictionary input " + newDictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
+ return getDictionaryInfo(dupDict);
+ }
+
+ Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(newDictInfo, dicts);
+
+ return trySaveNewDict(newDict, newDictInfo);
+ }
+
+ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException {
+
+ Object[] tmp = decideSourceData(cube, col, factColumnsPath);
+ String srcTable = (String) tmp[0];
+ String srcCol = (String) tmp[1];
+ int srcColIdx = (Integer) tmp[2];
+ ReadableTable inpTable = (ReadableTable) tmp[3];
+
+ DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature(), inpTable.getColumnDelimeter());
+
+ String dupDict = checkDupByInfo(dictInfo);
+ if (dupDict != null) {
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
+ return getDictionaryInfo(dupDict);
+ }
+
+ Dictionary<?> dict = DictionaryGenerator.buildDictionary(dictInfo, inpTable);
+
+ return trySaveNewDict(dict, dictInfo);
+ }
+
+ /**
+ * Get column origin
+ *
+ * @return 1. source table name
+ * 2. column name
+ * 3. column cardinal in source table
+ * 4. ReadableTable object
+ */
+ public Object[] decideSourceData(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException {
+ String srcTable;
+ String srcCol;
+ int srcColIdx;
+ ReadableTable table;
+ MetadataManager metaMgr = MetadataManager.getInstance(config);
+
+ // case of full table (dict on fact table)
+ if (cube == null) {
+ srcTable = col.getTable();
+ srcCol = col.getName();
+ srcColIdx = col.getColumn().getZeroBasedIndex();
+ int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount();
+ table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns);
+ return new Object[] { srcTable, srcCol, srcColIdx, table };
+ }
+
+ // Decide source data of dictionary:
+ // 1. If 'useDict' specifies pre-defined data set, use that
+ // 2. Otherwise find a lookup table to scan through
+
+ // Note FK on fact table is supported by scan the related PK on lookup
+ // table
+
+ String useDict = cube.getRowkey().getDictionary(col);
+
+ // normal case, source from lookup table
+ if ("true".equals(useDict) || "string".equals(useDict) || "number".equals(useDict) || "any".equals(useDict)) {
+ // FK on fact table, use PK from lookup instead
+ if (cube.isFactTable(col.getTable())) {
+ TblColRef pkCol = cube.findPKByFK(col);
+ if (pkCol != null)
+ col = pkCol; // scan the counterparty PK on lookup table
+ // instead
+ }
+ srcTable = col.getTable();
+ srcCol = col.getName();
+ srcColIdx = col.getColumn().getZeroBasedIndex();
+ if (cube.isFactTable(col.getTable())) {
+ table = new FileTable(factColumnsPath + "/" + col.getName(), -1);
+ } else {
+ table = new HiveTable(metaMgr, col.getTable());
+ }
+ }
+ // otherwise could refer to a data set, e.g. common_indicators.txt
+ // (LEGACY PATH, since distinct values are collected from fact table)
+ else {
+ String dictDataSetPath = unpackDataSet(this.config.getTempHDFSDir(), useDict);
+ if (dictDataSetPath == null)
+ throw new IllegalArgumentException("Unknown dictionary data set '" + useDict + "', referred from " + col);
+ srcTable = "PREDEFINED";
+ srcCol = useDict;
+ srcColIdx = 0;
+ table = new FileTable(dictDataSetPath, -1);
+ }
+
+ return new Object[] { srcTable, srcCol, srcColIdx, table };
+ }
+
+ private String unpackDataSet(String tempHDFSDir, String dataSetName) throws IOException {
+
+ InputStream in = this.getClass().getResourceAsStream("/com/kylinolap/dict/" + dataSetName + ".txt");
+ if (in == null) // data set resource not found
+ return null;
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ IOUtils.copy(in, buf);
+ in.close();
+ byte[] bytes = buf.toByteArray();
+
+ Path tmpDataSetPath = new Path(tempHDFSDir + "/dict/temp_dataset/" + dataSetName + "_" + bytes.length + ".txt");
+
+ FileSystem fs = HadoopUtil.getFileSystem(tempHDFSDir);
+ boolean writtenNewFile = false;
+ if (fs.exists(tmpDataSetPath) == false || fs.getFileStatus(tmpDataSetPath).getLen() != bytes.length) {
+ fs.mkdirs(tmpDataSetPath.getParent());
+ FSDataOutputStream out = fs.create(tmpDataSetPath);
+ IOUtils.copy(new ByteArrayInputStream(bytes), out);
+ out.close();
+ writtenNewFile = true;
+ }
+
+ String qualifiedPath = tmpDataSetPath.makeQualified(fs.getUri(), new Path("/")).toString();
+ if (writtenNewFile)
+ logger.info("Dictionary temp data set file written to " + qualifiedPath);
+ return qualifiedPath;
+ }
+
+ private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
+ if (existings == null)
+ return null;
+
+ TableSignature input = dictInfo.getInput();
+ for (String existing : existings) {
+ DictionaryInfo existingInfo = load(existing, false); // skip cache,
+ // direct
+ // load from
+ // store
+ if (input.equals(existingInfo.getInput()))
+ return existing;
+ }
+
+ return null;
+ }
+
+ private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
+ if (existings == null)
+ return null;
+
+ for (String existing : existings) {
+ logger.info("Checking dup dict :" + existing);
+ DictionaryInfo existingInfo = load(existing, true); // skip cache,
+ // direct load
+ // from store
+ if(existingInfo == null)
+ logger.info("existingInfo is null");
+
+ if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject()))
+ return existing;
+ }
+
+ return null;
+ }
+
+ public void removeDictionary(String resourcePath) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ store.deleteResource(resourcePath);
+ dictCache.remove(resourcePath);
+ }
+
+ public void removeDictionaries(String srcTable, String srcCol) throws IOException {
+ DictionaryInfo info = new DictionaryInfo();
+ info.setSourceTable(srcTable);
+ info.setSourceColumn(srcCol);
+
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ArrayList<String> existings = store.listResources(info.getResourceDir());
+ if (existings == null)
+ return;
+
+ for (String existing : existings)
+ removeDictionary(existing);
+ }
+
+ void save(DictionaryInfo dict) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ String path = dict.getResourcePath();
+ logger.info("Saving dictionary at " + path);
+ store.putResource(path, dict, DictionaryInfoSerializer.FULL_SERIALIZER);
+ }
+
+ DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+
+ DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER);
+
+ if (loadDictObj)
+ logger.debug("Loaded dictionary at " + resourcePath);
+
+ return info;
+ }
+
+}