[nifi-minifi-cpp] branch main updated: MINIFICPP-1707 Speed up linter

[sz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git


The following commit(s) were added to refs/heads/main by this push:
     new 5ec862c  MINIFICPP-1707 Speed up linter
5ec862c is described below

commit 5ec862c5783c681346fcd632bcc5d0152df33933
Author: Adam Markovics <nu...@gmail.com>
AuthorDate: Thu Jan 6 17:53:03 2022 +0100

    MINIFICPP-1707 Speed up linter
    
    Speedup is 19 to 5 seconds on my computer with using all cores:
        make -j12 linter
    
    Closes #1231
    Signed-off-by: Marton Szasz <sz...@apache.org>
---
 .github/workflows/ci.yml                   |  6 ++--
 CMakeLists.txt                             | 27 ++++++++++++----
 cmake/Extensions.cmake                     |  2 +-
 thirdparty/google-styleguide/run_linter.py | 52 ++++++++++++++++++++++++++----
 4 files changed, 69 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a933950..9a3da1a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,7 +39,7 @@ jobs:
       - name: test
         run: cd build && make test ARGS="--timeout 300 -j4 --output-on-failure"
       - name: linter
-        run: cd build && make linter
+        run: cd build && make -j4 linter
   windows_VS2019:
     name: "windows-vs2019"
     runs-on: windows-2019
@@ -86,7 +86,7 @@ jobs:
         run: cd ..\b && ctest --timeout 300 --parallel 8 -C Release --output-on-failure
         shell: cmd
       - name: linter
-        run: cd ..\b && msbuild linter.vcxproj
+        run: cd ..\b && msbuild -maxCpuCount linter.vcxproj
         shell: cmd
   ubuntu_20_04:
     name: "ubuntu-20.04"
@@ -123,7 +123,7 @@ jobs:
       - name: test
         run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure"
       - name: linter
-        run: cd build && make linter
+        run: cd build && make -j$(nproc) linter
       - name: shellcheck
         run: cd build && make shellcheck
       - id: flake8_check
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dec1c1a..a4d924a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -855,14 +855,27 @@ include(BuildDocs)
 
 
 # Create a custom build target that will run the linter.
+# Directories have their separate linter targets to be able to use better parallelization
 get_property(extensions GLOBAL PROPERTY EXTENSION-LINTERS)
-add_custom_target(linter
-	COMMAND python ${CMAKE_SOURCE_DIR}/thirdparty/google-styleguide/run_linter.py -q -i
-		${CMAKE_SOURCE_DIR}/libminifi/include/
-		${CMAKE_SOURCE_DIR}/libminifi/src/
-		${CMAKE_SOURCE_DIR}/libminifi/test/
-		${CMAKE_SOURCE_DIR}/encrypt-config/
-	DEPENDS ${extensions})
+set(root_linted_dirs libminifi/include libminifi/src libminifi/test encrypt-config)
+list(TRANSFORM root_linted_dirs PREPEND ${CMAKE_SOURCE_DIR}/)
+
+set(linted_dir_counter 1)
+set(root_linter_target_names "")
+
+foreach(linted_dir ${root_linted_dirs})
+	set(linter_target_name "root-linter-${linted_dir_counter}")
+	list(APPEND root_linter_target_names ${linter_target_name})
+	add_custom_target("${linter_target_name}"
+		COMMAND python3 ${CMAKE_SOURCE_DIR}/thirdparty/google-styleguide/run_linter.py -q -i
+		${linted_dir}
+	)
+	math(EXPR linted_dir_counter "${linted_dir_counter}+1")
+endforeach()
+
+# Main linter target that depends on every other
+add_custom_target(linter DEPENDS ${root_linter_target_names} ${extensions})
+
 
 if(NOT WIN32)
 add_custom_target(shellcheck
diff --git a/cmake/Extensions.cmake b/cmake/Extensions.cmake
index bd054ba..eeb0edb 100644
--- a/cmake/Extensions.cmake
+++ b/cmake/Extensions.cmake
@@ -115,7 +115,7 @@ macro(register_extension_linter target-name)
     get_property(extensions GLOBAL PROPERTY EXTENSION-LINTERS)
     set_property(GLOBAL APPEND PROPERTY EXTENSION-LINTERS "${target-name}")
     add_custom_target(${target-name}
-    COMMAND python ${CMAKE_SOURCE_DIR}/thirdparty/google-styleguide/run_linter.py -q -i ${CMAKE_CURRENT_LIST_DIR}/)
+    COMMAND python3 ${CMAKE_SOURCE_DIR}/thirdparty/google-styleguide/run_linter.py -q -i ${CMAKE_CURRENT_LIST_DIR}/)
   endif()
 endmacro()
 
diff --git a/thirdparty/google-styleguide/run_linter.py b/thirdparty/google-styleguide/run_linter.py
index 7f7f6fd..10d30cd 100644
--- a/thirdparty/google-styleguide/run_linter.py
+++ b/thirdparty/google-styleguide/run_linter.py
@@ -1,6 +1,9 @@
 import argparse
+import multiprocessing
 import os
 import cpplint
+import sys
+import math
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-i', '--includePaths', nargs="+", help='Run linter check in these directories')
@@ -9,10 +12,10 @@ args = parser.parse_args()
 
 list_of_files = list()
 for include_path in args.includePaths:
-  for (dir_path, dir_names, file_names) in os.walk(include_path):
-    for file_name in file_names:
-      if (".h" in file_name) or (".cpp" in file_name):
-        list_of_files += [os.path.join(dir_path, file_name)]
+    for (dir_path, dir_names, file_names) in os.walk(include_path):
+        for file_name in file_names:
+            if (".h" in file_name) or (".cpp" in file_name):
+                list_of_files += [os.path.join(dir_path, file_name)]
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 repository_path = os.path.abspath(os.path.join(script_dir, os.pardir, os.pardir))
@@ -20,7 +23,42 @@ repository_path = os.path.abspath(os.path.join(script_dir, os.pardir, os.pardir)
 arg_list = list()
 arg_list.append("--linelength=200")
 arg_list.append("--repository=" + repository_path)
-if (args.quiet):
-  arg_list.append("--quiet")
+if args.quiet:
+    arg_list.append("--quiet")
 
-cpplint.main(arg_list + list_of_files)
+
+def cpplint_main_wrapper(file_list):
+    try:
+        cpplint.main(arg_list + file_list)
+        return 0
+    except SystemExit as err:
+        return err.code
+
+
+def create_chunks(chunk_size, items):
+    """breaks up items to equal chunks, with last being shorter if items are not evenly divisible by chunk_size"""
+    chunk_list = []
+    chunk_begin = 0
+    chunk_end = chunk_size
+    for chunk_cnt in range(chunk_num):
+        chunk_list.append(items[chunk_begin:chunk_end])
+        chunk_begin += chunk_size
+        if chunk_begin >= len(list_of_files):
+            break
+        chunk_end += chunk_size
+        if chunk_end > len(list_of_files):
+            chunk_end = len(list_of_files)
+    return chunk_list
+
+
+if __name__ == '__main__':
+    chunk_num = multiprocessing.cpu_count()
+    chunk_max_size = math.ceil(len(list_of_files) / chunk_num)
+    chunks = create_chunks(chunk_max_size, list_of_files)
+    pool = multiprocessing.Pool(processes=chunk_num)
+    # execute in parallel
+    map_res = pool.map(cpplint_main_wrapper, chunks)
+    pool.close()
+    pool.join()
+    # pass failure exit code if one has failed
+    sys.exit(max(map_res))