[impala] 03/04: IMPALA-8101: Thrift 11 and ext-data-source compilation are always run

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit daa1bf9883e65adb82b11576b5ada4273bc9dd7f
Author: stakiar <ta...@gmail.com>
AuthorDate: Mon Jan 28 16:13:07 2019 -0600

    IMPALA-8101: Thrift 11 and ext-data-source compilation are always run
    
    Compilation of Thrift 11 Python code (IMPALA-7924) and of ext-data-source
    Thrift files (ErrorCodes.thrift, ExternalDataSource.thrift, Data.thrift,
    Status.thrift, Types.thrift) is run during every build, regardless of
    whether or not the .thrift files have changed. The issue is that the
    CMake custom command for compilation of these files points to a
    non-existent OUTPUT_FILE.
    
    This patch fixes Thrift 11 compilation by pointing the OUTPUT_FILE of
    each .thrift file to its corresponding __init__.py file. For
    compilation of ext-data-source, things are a bit tricky as we only run
    Java gen and it is difficult to map Java generated code to the
    corresponding .thrift files purely based on file names. Instead, for
    ext-data-source, this patch adds a dummy file under
    ext-data-source/api/target/tmp/generated-sources/ to track if a .thrift
    file has been compiled or not. A `mvn clean` of ext-data-source will
    delete all of these files and trigger re-compilation of the
    ext-data-source files.
    
    Change-Id: I52520e4b099c7bac5d088b4ba5d8a335495f727d
    Reviewed-on: http://gerrit.cloudera.org:8080/12290
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 common/thrift/CMakeLists.txt | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt
index 4d8646a..dd7e1f3 100644
--- a/common/thrift/CMakeLists.txt
+++ b/common/thrift/CMakeLists.txt
@@ -107,12 +107,19 @@ function(THRIFT_GEN_DS VAR)
     get_filename_component(ABS_THRIFT_FILE ${THRIFT_FILE} ABSOLUTE)
     get_filename_component(THRIFT_FILE_WE ${THRIFT_FILE} NAME_WE)
 
-    # This isn't the exact output file that's created, just a unique file
-    set(OUTPUT_FILE "${EXT_DS_OUTPUT_DIR}/${THRIFT_FILE_WE}.java")
+    # Create a dummy marker file to track if Thrift compilation of the current file has
+    # been run or not. This file is created only if Thrift compilation succeeds. Deletion
+    # of the file will re-trigger Thrift compilation for the current file.
+    set(OUTPUT_FILE_MESSAGE "CMake state file for Thrift compilation of"
+                            "${ABS_THRIFT_FILE}, deletion of this file will cause"
+                            "${THRIFT_FILE} to be recompiled during the next build")
+    set(OUTPUT_DIR "${EXT_DS_OUTPUT_DIR}/../target/tmp/generated-sources/")
+    set(OUTPUT_FILE "${OUTPUT_DIR}/${THRIFT_FILE_WE}.txt")
     list(APPEND ${VAR} ${OUTPUT_FILE})
     add_custom_command(
       OUTPUT ${OUTPUT_FILE}
-      COMMAND ${THRIFT_COMPILER} ${JAVA_EXT_DS_ARGS} ${THRIFT_FILE}
+      COMMAND ${THRIFT_COMPILER} ${JAVA_EXT_DS_ARGS} ${THRIFT_FILE} &&
+              mkdir -p ${OUTPUT_DIR} && echo ${OUTPUT_FILE_MESSAGE} > ${OUTPUT_FILE}
       DEPENDS ${ABS_THRIFT_FILE}
       COMMENT "Running thrift compiler for ext-data-source on ${THRIFT_FILE}"
       VERBATIM
@@ -141,7 +148,14 @@ function(THRIFT11_GEN VAR)
     get_filename_component(THRIFT_FILE_WE ${THRIFT_FILE} NAME_WE)
 
     # This isn't the exact output file that's created, just a unique file
-    set(OUTPUT_FILE "${THRIFT11_PYTHON_OUTPUT_DIR}/${THRIFT_FILE_WE}.py")
+    set(OUTPUT_FILE "${THRIFT11_PYTHON_OUTPUT_DIR}/gen-py/${THRIFT_FILE_WE}/__init__.py")
+
+    # We have to special case beeswax.thrift because its Python namespace is beeswaxd
+    # instead of beeswax
+    if (${THRIFT_FILE_WE} STREQUAL "beeswax")
+      set(OUTPUT_FILE "${THRIFT11_PYTHON_OUTPUT_DIR}/gen-py/beeswaxd/__init__.py")
+    endif(${THRIFT_FILE_WE} STREQUAL "beeswax")
+
     list(APPEND ${VAR} ${OUTPUT_FILE})
     add_custom_command(
       OUTPUT ${OUTPUT_FILE}