You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by ji...@apache.org on 2022/08/18 06:48:13 UTC
[incubator-pegasus] 01/05: refactor(build): remove 'rdsn' from build and test commands (#1099)
This is an automated email from the ASF dual-hosted git repository.
jiashuo pushed a commit to branch v2.4
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 4c35d4153af63981f8a923cf7eb889809d37d1fb
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Fri Aug 12 15:42:07 2022 +0800
refactor(build): remove 'rdsn' from build and test commands (#1099)
---
.github/workflows/lint_and_test_cpp.yaml | 373 ++++++++++++++---------
.gitignore | 3 +-
CMakeLists.txt | 15 +-
cmake_modules/BaseFunctions.cmake | 11 +
run.sh | 185 ++++-------
scripts/config_hdfs.sh | 2 +-
src/CMakeLists.txt | 6 +-
src/client_lib/CMakeLists.txt | 10 +-
src/geo/test/CMakeLists.txt | 2 +-
src/geo/test/config.ini | 2 +-
src/geo/test/geo_test.cpp | 11 +
src/geo/test/main.cpp | 1 +
src/geo/test/{CMakeLists.txt => run.sh} | 38 +--
src/rdsn/CMakeLists.txt | 45 ---
src/rdsn/src/CMakeLists.txt | 29 ++
src/rdsn/src/meta/CMakeLists.txt | 2 +
src/rdsn/src/meta/meta_service.cpp | 83 -----
src/rdsn/src/meta/meta_service.h | 83 +++++
src/rdsn/src/meta/test/CMakeLists.txt | 2 +-
src/rdsn/src/meta/test/run.sh | 14 +
src/rdsn/src/replica/test/replica_test.cpp | 3 +-
src/rdsn/src/runtime/rpc/rpc_message.cpp | 2 +-
src/rdsn/src/runtime/rpc/thrift_message_parser.h | 2 +-
src/rdsn/src/runtime/task/task_spec.cpp | 8 +-
src/server/hotkey_collector.cpp | 4 +
src/test/function_test/test_batch_get.cpp | 43 +--
26 files changed, 514 insertions(+), 465 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index 2a8cace2d..88aa2db9d 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -21,7 +21,7 @@
# it's recommended to create a PR against the ci-test branch to test if it works
# as expected.
-name: Lint and Unit Test for Cpp
+name: Cpp CI
on:
# run on each pull request
@@ -36,13 +36,9 @@ on:
# for manually triggering workflow
workflow_dispatch:
-defaults:
- run:
- shell: bash
-
jobs:
cpp_clang_format_linter:
- name: Lint Cpp code
+ name: Lint
runs-on: ubuntu-latest
container:
image: apache/pegasus:clang-format-3.9
@@ -54,22 +50,20 @@ jobs:
- name: clang-format
run: ./scripts/run-clang-format.py --clang-format-executable clang-format-3.9 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
- test_rdsn_Release:
- name: Test rdsn with Release
+ build_Release:
+ name: Build Release
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
container:
image: apache/pegasus:thirdparties-bin-ubuntu1804
- env:
- CCACHE_DIR: /tmp/ccache/pegasus
- CCACHE_MAXSIZE: 10G
- volumes:
- # Place ccache compilation intermediate results in host memory, that's shared among containers.
- - /tmp/ccache/pegasus:/tmp/ccache/pegasus
- # Read docs at https://docs.docker.com/storage/tmpfs/ for more details of using tmpfs in docker.
- options: --mount type=tmpfs,destination=/tmp/pegasus --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v2
+ - name: Setup cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ /github/home/.ccache
+ key: release_ccache
- uses: dorny/paths-filter@v2
id: changes
with:
@@ -78,8 +72,8 @@ jobs:
- 'cmake_modules/**'
- 'CMakeLists.txt'
- 'run.sh'
- rdsn:
- - 'src/rdsn/**'
+ src:
+ - 'src/**'
thirdparty:
- 'thirdparty/**'
- name: Unpack prebuilt third-parties
@@ -91,81 +85,98 @@ jobs:
run: |
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -B build/
- cmake --build build/ -j $(($(nproc)/2+1))
+ cmake --build build/ -j $(nproc)
- name: Compilation
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build -c --rdsn --test --skip_thirdparty -j $(($(nproc)/2+1))
- - name: Unit Testing
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
+ if: steps.changes.outputs.base == 'true' || steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
run: |
- export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
- ulimit -s unlimited
- ./run.sh test --rdsn
+ ccache -p
+ ccache -z
+ ./run.sh build --test --skip_thirdparty -j $(nproc) -t release
+ ccache -s
+ - name: Pack Server
+ if: steps.changes.outputs.base == 'true' || steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
+ run: ./run.sh pack_server
+ - name: Pack Tools
+ if: steps.changes.outputs.base == 'true' || steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
+ run: ./run.sh pack_tools
+ - name: Tar files
+ run: |
+ rm -rf thirdparty
+ tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ - name: Upload Artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: release_artifact_${{ github.sha }}
+ path: release__builder.tar
- test_rdsn_ASAN:
- name: Test rdsn with ASAN
- needs: cpp_clang_format_linter
+ test_Release:
+ name: Test Release
+ strategy:
+ fail-fast: false
+ matrix:
+ test_module:
+ - base_test
+ - dsn_aio_test
+ - dsn_block_service_test
+ - dsn.failure_detector.tests
+ - dsn_http_test
+ - dsn_meta_state_tests
+ - dsn.meta.test
+ - dsn_nfs_test
+ - dsn_perf_counter_test
+ - dsn_replica_backup_test
+ - dsn_replica_bulk_load_test
+ - dsn_replica_dup_test
+ - dsn_replica_split_test
+ - dsn.replica.test
+ - dsn_replication_common_test
+ - dsn.replication.simple_kv
+ - dsn.rep_tests.simple_kv
+ - dsn_runtime_tests
+ - dsn_utils_tests
+ - dsn.zookeeper.tests
+ - pegasus_function_test
+ - pegasus_geo_test
+ - pegasus_rproxy_test
+ - pegasus_unit_test
+ needs: build_Release
runs-on: ubuntu-latest
container:
image: apache/pegasus:thirdparties-bin-ubuntu1804
- env:
- CCACHE_DIR: /tmp/ccache/pegasus
- CCACHE_MAXSIZE: 10G
- volumes:
- # Place ccache compilation intermediate results in host memory, that's shared among containers.
- - /tmp/ccache/pegasus:/tmp/ccache/pegasus
- # Read docs at https://docs.docker.com/storage/tmpfs/ for more details of using tmpfs in docker.
- options: --mount type=tmpfs,destination=/tmp/pegasus --cap-add=SYS_PTRACE
+ options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v2
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- base:
- - 'cmake_modules/**'
- - 'CMakeLists.txt'
- - 'run.sh'
- rdsn:
- - 'src/rdsn/**'
- thirdparty:
- - 'thirdparty/**'
- name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
run: unzip /root/thirdparties-bin.zip -d ./thirdparty
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
+ - name: Download Artifact
+ uses: actions/download-artifact@v3
+ with:
+ name: release_artifact_${{ github.sha }}
+ path: .
+ - name: Tar files
run: |
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -B build/
- cmake --build build/ -j $(($(nproc)/2+1))
- - name: Compilation
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build -c --rdsn --test --sanitizer address --skip_thirdparty --disable_gperf -j $(($(nproc)/2+1))
+ tar -zxvf release__builder.tar
- name: Unit Testing
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./run.sh test --rdsn
+ ./scripts/config_hdfs.sh
+ ./run.sh test --on_travis -m ${{ matrix.test_module }}
- test_rdsn_LSAN:
- name: Test rdsn with LSAN
+ build_ASAN:
+ name: Build ASAN
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
container:
image: apache/pegasus:thirdparties-bin-ubuntu1804
- env:
- CCACHE_DIR: /tmp/ccache/pegasus
- CCACHE_MAXSIZE: 10G
- volumes:
- # Place ccache compilation intermediate results in host memory, that's shared among containers.
- - /tmp/ccache/pegasus:/tmp/ccache/pegasus
- # Read docs at https://docs.docker.com/storage/tmpfs/ for more details of using tmpfs in docker.
- options: --mount type=tmpfs,destination=/tmp/pegasus --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v2
+ - name: Setup cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ /github/home/.ccache
+ key: asan_ccache
- uses: dorny/paths-filter@v2
id: changes
with:
@@ -174,8 +185,8 @@ jobs:
- 'cmake_modules/**'
- 'CMakeLists.txt'
- 'run.sh'
- rdsn:
- - 'src/rdsn/**'
+ src:
+ - 'src/**'
thirdparty:
- 'thirdparty/**'
- name: Unpack prebuilt third-parties
@@ -187,77 +198,92 @@ jobs:
run: |
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -B build/
- cmake --build build/ -j $(($(nproc)/2+1))
+ cmake --build build/ -j $(nproc)
- name: Compilation
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build -c --rdsn --test --sanitizer leak --skip_thirdparty --disable_gperf -j $(($(nproc)/2+1))
- - name: Unit Testing
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
+ if: steps.changes.outputs.base == 'true' || steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
run: |
- export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
- ulimit -s unlimited
- ./run.sh test --rdsn
+ ccache -p
+ ccache -z
+ ./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc)
+ ccache -s
+ - name: Tar files
+ run: |
+ rm -rf thirdparty
+ tar -zcvhf release_address_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ - name: Upload Artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: release_address_artifact_${{ github.sha }}
+ path: release_address_builder.tar
- test_rdsn_UBSAN:
- name: Test rdsn with UBSAN
- needs: cpp_clang_format_linter
+ test_ASAN:
+ name: Test ASAN
+ strategy:
+ fail-fast: false
+ matrix:
+ test_module:
+ - base_test
+ - dsn_aio_test
+ - dsn_block_service_test
+ - dsn.failure_detector.tests
+ - dsn_http_test
+ - dsn_meta_state_tests
+ - dsn.meta.test
+ - dsn_nfs_test
+ - dsn_perf_counter_test
+ - dsn_replica_backup_test
+ - dsn_replica_bulk_load_test
+ - dsn_replica_dup_test
+ - dsn_replica_split_test
+ - dsn.replica.test
+ - dsn_replication_common_test
+ - dsn.replication.simple_kv
+ - dsn.rep_tests.simple_kv
+ - dsn_runtime_tests
+ - dsn_utils_tests
+ - dsn.zookeeper.tests
+ - pegasus_function_test
+ - pegasus_geo_test
+ - pegasus_rproxy_test
+ - pegasus_unit_test
+ needs: build_ASAN
runs-on: ubuntu-latest
container:
image: apache/pegasus:thirdparties-bin-ubuntu1804
- env:
- CCACHE_DIR: /tmp/ccache/pegasus
- CCACHE_MAXSIZE: 10G
- volumes:
- # Place ccache compilation intermediate results in host memory, that's shared among containers.
- - /tmp/ccache/pegasus:/tmp/ccache/pegasus
- # Read docs at https://docs.docker.com/storage/tmpfs/ for more details of using tmpfs in docker.
- options: --mount type=tmpfs,destination=/tmp/pegasus --cap-add=SYS_PTRACE
+ options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v2
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- base:
- - 'cmake_modules/**'
- - 'CMakeLists.txt'
- - 'run.sh'
- rdsn:
- - 'src/rdsn/**'
- thirdparty:
- - 'thirdparty/**'
- name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
run: unzip /root/thirdparties-bin.zip -d ./thirdparty
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
+ - name: Download Artifact
+ uses: actions/download-artifact@v3
+ with:
+ name: release_address_artifact_${{ github.sha }}
+ path: .
+ - name: Tar files
run: |
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -B build/
- cmake --build build/ -j $(($(nproc)/2+1))
- - name: Compilation
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build -c --rdsn --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(($(nproc)/2+1))
+ tar -zxvf release_address_builder.tar
- name: Unit Testing
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.rdsn == 'true' || steps.changes.outputs.thirdparty == 'true'
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./run.sh test --rdsn
+ ./scripts/config_hdfs.sh
+ ./run.sh test --on_travis -m ${{ matrix.test_module }}
- # TODO(yingchun): also need to test ASAN, LSAN, UBSAN
- test_pegasus:
- name: Test Pegasus
+ build_UBSAN:
+ name: Build UBSAN
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
container:
image: apache/pegasus:thirdparties-bin-ubuntu1804
- defaults:
- run:
- shell: bash
steps:
- uses: actions/checkout@v2
+ - name: Setup cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ /github/home/.ccache
+ key: ubsan_ccache
- uses: dorny/paths-filter@v2
id: changes
with:
@@ -266,7 +292,7 @@ jobs:
- 'cmake_modules/**'
- 'CMakeLists.txt'
- 'run.sh'
- pegasus:
+ src:
- 'src/**'
thirdparty:
- 'thirdparty/**'
@@ -279,24 +305,80 @@ jobs:
run: |
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -B build/
- cmake --build build/ -j $(($(nproc)/2+1))
+ cmake --build build/ -j $(nproc)
- name: Compilation
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.pegasus == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build --test -c -j $(($(nproc)/2+1)) --skip_thirdparty
- - name: Packaging Server
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.pegasus == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh pack_server
- - name: Packaging Tools
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.pegasus == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh pack_tools
+ if: steps.changes.outputs.base == 'true' || steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
+ run: |
+ ccache -p
+ ccache -z
+ ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc)
+ ccache -s
+ - name: Tar files
+ run: |
+ rm -rf thirdparty
+ tar -zcvhf release_undefined_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ - name: Upload Artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: release_undefined_artifact_${{ github.sha }}
+ path: release_undefined_builder.tar
+
+ test_UBSAN:
+ name: Test UBSAN
+ strategy:
+ fail-fast: false
+ matrix:
+ test_module:
+ - base_test
+ - dsn_aio_test
+ - dsn_block_service_test
+ - dsn.failure_detector.tests
+ - dsn_http_test
+ - dsn_meta_state_tests
+ - dsn.meta.test
+ - dsn_nfs_test
+ - dsn_perf_counter_test
+ - dsn_replica_backup_test
+ - dsn_replica_bulk_load_test
+ - dsn_replica_dup_test
+ - dsn_replica_split_test
+ - dsn.replica.test
+ - dsn_replication_common_test
+ - dsn.replication.simple_kv
+ - dsn.rep_tests.simple_kv
+ - dsn_runtime_tests
+ - dsn_utils_tests
+ - dsn.zookeeper.tests
+ - pegasus_function_test
+ - pegasus_geo_test
+ - pegasus_rproxy_test
+ - pegasus_unit_test
+ needs: build_UBSAN
+ runs-on: ubuntu-latest
+ container:
+ image: apache/pegasus:thirdparties-bin-ubuntu1804
+ options: --cap-add=SYS_PTRACE
+ steps:
+ - uses: actions/checkout@v2
+ - name: Unpack prebuilt third-parties
+ run: unzip /root/thirdparties-bin.zip -d ./thirdparty
+ - name: Download Artifact
+ uses: actions/download-artifact@v3
+ with:
+ name: release_undefined_artifact_${{ github.sha }}
+ path: .
+ - name: Tar files
+ run: |
+ tar -zxvf release_undefined_builder.tar
- name: Unit Testing
- if: steps.changes.outputs.base == 'true' || steps.changes.outputs.pegasus == 'true' || steps.changes.outputs.thirdparty == 'true'
run: |
- source ./scripts/config_hdfs.sh
- ./run.sh test --on_travis
+ export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
+ ulimit -s unlimited
+ ./scripts/config_hdfs.sh
+ ./run.sh test --on_travis -m ${{ matrix.test_module }}
build_pegasus_on_macos:
- name: Build Pegasus on macOS
+ name: macOS
needs: cpp_clang_format_linter
runs-on: macos-12
env:
@@ -305,18 +387,21 @@ jobs:
- name: Install Softwares
run: |
# Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md
+ brew install ccache
brew install snappy
brew install lz4
brew install zstd
brew install openssl@1.1
- uses: actions/checkout@v2
- name: Setup cache
- id: cache
uses: actions/cache@v3
with:
path: |
- thirdparty
- key: macos-12-thirdparty-bins
+ /Users/runner/Library/Caches/ccache
+ key: macos_build_ccache
+ restore-keys: |
+ macos_build_ccache
+ macos-12-thirdparty-bins
- name: Build thirdparty
working-directory: thirdparty
run: |
@@ -328,10 +413,14 @@ jobs:
id: changes
with:
filters: |
- pegasus:
+ src:
- 'src/**'
thirdparty:
- 'thirdparty/**'
- name: Compilation
- if: steps.changes.outputs.pegasus == 'true' || steps.changes.outputs.thirdparty == 'true'
- run: ./run.sh build --test --skip_thirdparty -c -j $(sysctl -n hw.physicalcpu)
+ if: steps.changes.outputs.src == 'true' || steps.changes.outputs.thirdparty == 'true'
+ run: |
+ ccache -p
+ ccache -z
+ ./run.sh build --test --skip_thirdparty -j $(sysctl -n hw.physicalcpu)
+ ccache -s
diff --git a/.gitignore b/.gitignore
index 934806e33..c7cc60c70 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,11 +18,10 @@
src/.matchfile
src/base/rrdb_types.cpp
src/include/rrdb/rrdb_types.h
-src/rdsn/builder/
src/rdsn/include/dsn/cpp/serialization_helper/dsn.layer2_types.h
src/rdsn/src/runtime/dsn.layer2_types.cpp
src/runtime
-src/builder
+src/*builder
onebox/
.zk_install/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 82715a11d..036041f79 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -44,6 +44,9 @@ message(STATUS "THIRDPARTY_INSTALL_DIR = ${THIRDPARTY_INSTALL_DIR}")
set(DSN_ROOT ${PROJECT_ROOT}/DSN_ROOT)
message(STATUS "DSN_ROOT = ${DSN_ROOT}")
+set(BUILD_DIR ${PROJECT_ROOT}/src/builder)
+message(STATUS "BUILD_DIR = ${BUILD_DIR}")
+
option(BUILD_TEST "build unit test" ON)
message(STATUS "BUILD_TEST = ${BUILD_TEST}")
@@ -66,14 +69,4 @@ if(USE_JEMALLOC)
set(JEMALLOC_LIB_TYPE "SHARED")
endif()
-option(BUILD_RDSN "Whether to build rdsn project" ON)
-option(BUILD_PEGASUS "Whether to build Pegasus project" ON)
-
-# TODO(yingchun): we will refactor the followng code later
-if(BUILD_RDSN)
- add_subdirectory(src/rdsn)
-endif()
-
-if(BUILD_PEGASUS)
- add_subdirectory(src)
-endif()
+add_subdirectory(src)
diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake
index f833125bc..a301d87ff 100644
--- a/cmake_modules/BaseFunctions.cmake
+++ b/cmake_modules/BaseFunctions.cmake
@@ -372,6 +372,17 @@ function(dsn_common_setup)
message(FATAL_ERROR "In-source builds are not allowed.")
endif()
+ find_program(CCACHE "ccache")
+ if(CCACHE)
+ set(CMAKE_C_COMPILER_LAUNCHER ${CCACHE})
+ set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE})
+ message(STATUS "CCACHE: ${CCACHE}")
+
+ set(ENV{CCACHE_COMPRESS} "true")
+ set(ENV{CCACHE_COMPRESSLEVEL} "6")
+ set(ENV{CCACHE_MAXSIZE} "1024M")
+ endif(CCACHE)
+
if(NOT DEFINED DSN_BUILD_RUNTIME)
set(DSN_BUILD_RUNTIME FALSE)
endif()
diff --git a/run.sh b/run.sh
index f2dce0005..06606e678 100755
--- a/run.sh
+++ b/run.sh
@@ -75,9 +75,8 @@ function usage_build()
echo "Options for subcommand 'build':"
echo " -h|--help print the help info"
echo " -t|--type build type: debug|release, default is release"
- echo " -s|--serialize serialize type: dsn|thrift|proto, default is thrift"
- echo " -c|--clear clear rdsn/pegasus before building, not clear thirdparty"
- echo " --clear_thirdparty clear thirdparty/rdsn/pegasus before building"
+ echo " -c|--clear clear pegasus before building, not clear thirdparty"
+ echo " --clear_thirdparty clear thirdparty/pegasus before building"
echo " --compiler specify c and cxx compiler, sperated by ','"
echo " e.g., \"gcc,g++\" or \"clang-3.9,clang++-3.9\""
echo " default is \"gcc,g++\""
@@ -91,7 +90,6 @@ function usage_build()
type: address|leak|thread|undefined"
echo " --skip_thirdparty whether to skip building thirdparties, default no"
echo " --enable_rocksdb_portable build a portable rocksdb binary"
- echo " --rdsn whether to build rdsn module only"
echo " --test whether to build test binaries"
}
@@ -121,7 +119,6 @@ function run_build()
ROCKSDB_PORTABLE=OFF
USE_JEMALLOC=OFF
BUILD_TEST=OFF
- ONLY_RDSN=NO
while [[ $# > 0 ]]; do
key="$1"
case $key in
@@ -186,9 +183,6 @@ function run_build()
--test)
BUILD_TEST=ON
;;
- --rdsn)
- ONLY_RDSN=YES
- ;;
*)
echo "ERROR: unknown option \"$key\""
echo
@@ -252,64 +246,45 @@ function run_build()
cd ..
fi
- echo "INFO: Start build rdsn..."
- BUILD_DIR="$ROOT/src/rdsn/builder"
+ echo "INFO: start build Pegasus..."
+ BUILD_DIR="${ROOT}/src/${BUILD_TYPE}_${SANITIZER}_builder"
if [ "$CLEAR" == "YES" ]; then
echo "Clear $BUILD_DIR ..."
rm -rf $BUILD_DIR
fi
+
+ pushd ${ROOT}
+ echo "Gen thrift"
+ # TODO(yingchun): should be optimized
+ python3 $ROOT/scripts/compile_thrift.py
+ sh ${ROOT}/scripts/recompile_thrift.sh
+
+ mkdir -p ${DSN_ROOT}
if [ ! -d "$BUILD_DIR" ]; then
mkdir -p $BUILD_DIR
- echo "Gen rdsn thrift"
- python3 $ROOT/scripts/compile_thrift.py
-
- echo "Running cmake rdsn ..."
+ echo "Running cmake Pegasus..."
pushd $BUILD_DIR
CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_TEST=${BUILD_TEST}"
- cmake ../../.. -DCMAKE_INSTALL_PREFIX=$BUILD_DIR/output $CMAKE_OPTIONS -DBUILD_RDSN=ON -DBUILD_PEGASUS=OFF
+ cmake ../.. -DCMAKE_INSTALL_PREFIX=$BUILD_DIR/output $CMAKE_OPTIONS
exit_if_fail $?
fi
- echo "[$(date)] Building rdsn ..."
- pushd $BUILD_DIR
- make install $MAKE_OPTIONS
- exit_if_fail $?
-
- if [ "$ONLY_RDSN" == "YES" ]; then
- exit 0
- fi
-
- echo "INFO: start build Pegasus..."
- BUILD_DIR="$ROOT/src/builder"
- if [ "$CLEAR" == "YES" ]; then
- echo "Clear $BUILD_DIR ..."
- rm -rf $BUILD_DIR
+ echo "Gen git_commit.h ..."
+ pushd "$ROOT/src"
+ PEGASUS_GIT_COMMIT="non-git-repo"
+ if git rev-parse HEAD; then # this is a git repo
+ PEGASUS_GIT_COMMIT=$(git rev-parse HEAD)
fi
- pushd ${ROOT}
- if [ ! -d "$BUILD_DIR" ]; then
- mkdir -p $BUILD_DIR
-
- echo "Gen git_commit.h ..."
- pushd "$ROOT/src"
- PEGASUS_GIT_COMMIT="non-git-repo"
- if git rev-parse HEAD; then # this is a git repo
- PEGASUS_GIT_COMMIT=$(git rev-parse HEAD)
- fi
- echo "PEGASUS_GIT_COMMIT=${PEGASUS_GIT_COMMIT}"
- GIT_COMMIT_FILE=$ROOT/src/include/pegasus/git_commit.h
- echo "Generating $GIT_COMMIT_FILE..."
- echo "#pragma once" >$GIT_COMMIT_FILE
- echo "#define PEGASUS_GIT_COMMIT \"$PEGASUS_GIT_COMMIT\"" >>$GIT_COMMIT_FILE
-
- echo "Gen pegasus thrift"
- sh ${ROOT}/scripts/recompile_thrift.sh
+ echo "PEGASUS_GIT_COMMIT=${PEGASUS_GIT_COMMIT}"
+ GIT_COMMIT_FILE=$ROOT/src/include/pegasus/git_commit.h
+ echo "Generating $GIT_COMMIT_FILE..."
+ echo "#pragma once" >$GIT_COMMIT_FILE
+ echo "#define PEGASUS_GIT_COMMIT \"$PEGASUS_GIT_COMMIT\"" >>$GIT_COMMIT_FILE
- echo "Running cmake Pegasus..."
- pushd $BUILD_DIR
- cmake ../.. -DCMAKE_INSTALL_PREFIX=$BUILD_DIR/output $CMAKE_OPTIONS -DBUILD_RDSN=OFF -DBUILD_PEGASUS=ON
- exit_if_fail $?
- fi
+ # rebuild link
+ rm -f ${ROOT}/src/builder
+ ln -s ${BUILD_DIR} ${ROOT}/src/builder
echo "[$(date)] Building Pegasus ..."
pushd $BUILD_DIR
@@ -334,14 +309,12 @@ function usage_test()
echo " if not set, then run all tests"
echo " -k|--keep_onebox whether keep the onebox after the test[default false]"
echo " --on_travis run tests on travis without some time-cosuming function tests"
- echo " --rdsn whether to test rdsn module only"
}
function run_test()
{
local test_modules=""
local clear_flags="1"
local on_travis=""
- local only_rdsn="no"
local enable_gcov="no"
while [[ $# > 0 ]]; do
key="$1"
@@ -360,9 +333,6 @@ function run_test()
--on_travis)
on_travis="--on_travis"
;;
- --rdsn)
- only_rdsn="yes"
- ;;
--enable_gcov)
enable_gcov="yes"
;;
@@ -383,42 +353,13 @@ function run_test()
mkdir -p $REPORT_DIR
fi
- if [ "$only_rdsn" == "yes" ]; then
- BUILD_DIR=$ROOT/src/rdsn/builder
- run_rdsn_test
- exit 0
- fi
-
BUILD_DIR=$ROOT/src/builder
- run_pegasus_test
-
- echo "Test finish time: `date`"
- finish_time=`date +%s`
- used_time=$((finish_time-start_time))
- echo "Test elapsed time: $((used_time/60))m $((used_time%60))s"
-
- if [ "$enable_gcov" == "yes" ]; then
- echo "Generating gcov report..."
- cd $ROOT
- mkdir -p "$ROOT/gcov_report"
-
- echo "Running gcovr to produce HTML code coverage report."
- $BUILD_DIR
- gcovr --html --html-details -r $ROOT --object-directory=$BUILD_DIR \
- -o $GCOV_DIR/index.html
- if [ $? -ne 0 ]; then
- exit 1
- fi
- fi
-}
-
-function run_pegasus_test()
-{
if [ "$test_modules" == "" ]; then
- test_modules="pegasus_unit_test,pegasus_function_test"
+ test_modules="dsn_runtime_tests,dsn_utils_tests,dsn_perf_counter_test,dsn.zookeeper.tests,dsn_aio_test,dsn.failure_detector.tests,dsn_meta_state_tests,dsn_nfs_test,dsn_block_service_test,dsn.replication.simple_kv,dsn.rep_tests.simple_kv,dsn.meta.test,dsn.replica.test,dsn_http_test,dsn_replica_dup_test,dsn_replica_backup_test,dsn_replica_bulk_load_test,dsn_replica_split_test,pegasus_unit_test,pegasus_function_test"
fi
echo "test_modules=$test_modules"
+ # download bulk load test data
if [[ "$test_modules" =~ "pegasus_function_test" && "$on_travis" == "" && ! -d "$ROOT/src/test/function_test/pegasus-bulk-load-function-test-files" ]]; then
echo "Start to download files used for bulk load function test"
wget "https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-bulk-load-function-test-files.zip"
@@ -427,19 +368,28 @@ function run_pegasus_test()
echo "Prepare files used for bulk load function test succeed"
fi
- # restart onebox
- run_clear_onebox
- if ! run_start_onebox -w; then
- echo "ERROR: unable to continue on testing because starting onebox failed"
- exit 1
- fi
+ # restart zk
+ run_stop_zk
+ run_start_zk
- sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g" $ROOT/src/builder/src/server/test/config.ini
+ # restart onebox when test pegasus
+ if [[ "$test_modules" =~ "pegasus" ]]; then
+ run_clear_onebox
+ if ! run_start_onebox -w; then
+ echo "ERROR: unable to continue on testing because starting onebox failed"
+ exit 1
+ fi
+ sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g" $ROOT/src/builder/src/server/test/config.ini
+ fi
for module in `echo $test_modules | sed 's/,/ /g'`; do
echo "====================== run $module =========================="
pushd $ROOT/src/builder/bin/$module
- REPORT_DIR=$REPORT_DIR ./run.sh $on_travis
+ parms=""
+ if [[ "$module" =~ "pegasus" ]]; then
+ parms=$on_travis
+ fi
+ REPORT_DIR=$REPORT_DIR ./run.sh $parms
if [ $? != 0 ]; then
echo "run test \"$module\" in `pwd` failed"
exit 1
@@ -447,34 +397,33 @@ function run_pegasus_test()
popd
done
- if [ "$clear_flags" == "1" ]; then
- run_clear_onebox
- fi
-}
-function run_rdsn_test()
-{
- if [ "$test_modules" == "" ]; then
- test_modules="dsn_runtime_tests,dsn_utils_tests,dsn_perf_counter_test,dsn.zookeeper.tests,dsn_aio_test,dsn.failure_detector.tests,dsn_meta_state_tests,dsn_nfs_test,dsn_block_service_test,dsn.replication.simple_kv,dsn.rep_tests.simple_kv,dsn.meta.test,dsn.replica.test,dsn_http_test,dsn_replica_dup_test,dsn_replica_backup_test,dsn_replica_bulk_load_test,dsn_replica_split_test"
+ # clear onebox if needed
+ if [[ "$test_modules" =~ "pegasus" ]]; then
+ if [ "$clear_flags" == "1" ]; then
+ run_clear_onebox
+ fi
fi
- echo "test_modules=$test_modules"
- # restart zk
- run_stop_zk
- run_start_zk
+ echo "Test finish time: `date`"
+ finish_time=`date +%s`
+ used_time=$((finish_time-start_time))
+ echo "Test elapsed time: $((used_time/60))m $((used_time%60))s"
- for module in `echo $test_modules | sed 's/,/ /g'`; do
- echo "====================== run $module =========================="
- pushd $ROOT/src/rdsn/builder/bin/$module
- REPORT_DIR=$REPORT_DIR ./run.sh
- if [ $? != 0 ]; then
- echo "run test \"$module\" in `pwd` failed"
+ if [ "$enable_gcov" == "yes" ]; then
+ echo "Generating gcov report..."
+ cd $ROOT
+ mkdir -p "$ROOT/gcov_report"
+
+ echo "Running gcovr to produce HTML code coverage report."
+ $BUILD_DIR
+ gcovr --html --html-details -r $ROOT --object-directory=$BUILD_DIR \
+ -o $GCOV_DIR/index.html
+ if [ $? -ne 0 ]; then
exit 1
fi
- popd
- done
-
- echo "Test succeed"
+ fi
}
+
#####################
## start_zk
#####################
diff --git a/scripts/config_hdfs.sh b/scripts/config_hdfs.sh
index 32aa0375c..5ff58d542 100755
--- a/scripts/config_hdfs.sh
+++ b/scripts/config_hdfs.sh
@@ -46,7 +46,7 @@ HDFS_TAR_MD5_VALUE="b30b409bb69185003b3babd1504ba224"
if [ ! -f $HDFS_ROOT ]; then
echo "Downloading hadoop..."
download_url="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/hadoop-2.8.4.tar.gz"
- if ! wget -T 5 -t 1 $download_url; then
+ if ! wget -T 10 -t 5 $download_url; then
echo "ERROR: download hadoop failed"
exit 1
fi
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ebbdcf11c..f2aa6bd8b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -27,15 +27,15 @@ dsn_common_setup()
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
-include_directories(${DSN_ROOT}/include)
-
-link_directories(${DSN_ROOT}/lib)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/rdsn/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/rdsn/include/dsn/cpp/serialization_helper)
# Users don't have to configure CMAKE_INSTALL_PREFIX unless they want to customize
# the destination.
set(CMAKE_INSTALL_PREFIX ${DSN_ROOT} CACHE STRING "" FORCE)
message (STATUS "pegasus Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX})
+add_subdirectory(rdsn/src)
add_subdirectory(base)
add_subdirectory(reporter)
add_subdirectory(base/test)
diff --git a/src/client_lib/CMakeLists.txt b/src/client_lib/CMakeLists.txt
index cb65778ef..76f93b5e5 100644
--- a/src/client_lib/CMakeLists.txt
+++ b/src/client_lib/CMakeLists.txt
@@ -31,19 +31,19 @@ target_include_directories(pegasus_client_impl_objects PUBLIC $<TARGET_PROPERTY:
set(pegasus_client_static_lib ${CMAKE_CURRENT_BINARY_DIR}/libpegasus_client_static.a)
if (APPLE)
add_custom_target(pre_combine_lib
- COMMAND ar -x ${DSN_ROOT}/lib/libdsn_utils.a)
+ COMMAND ar -x ${BUILD_DIR}/src/rdsn/src/util/libdsn_utils.a)
endif()
add_custom_target(combine_lib
- COMMAND ar -x ${DSN_ROOT}/lib/libdsn_runtime.a
- COMMAND ar -x ${DSN_ROOT}/lib/libdsn_client.a
- COMMAND ar -x ${DSN_ROOT}/lib/libdsn_replication_common.a
+ COMMAND ar -x ${BUILD_DIR}/src/rdsn/src/runtime/libdsn_runtime.a
+ COMMAND ar -x ${BUILD_DIR}/src/rdsn/src/client/libdsn_client.a
+ COMMAND ar -x ${BUILD_DIR}/src/rdsn/src/common/libdsn_replication_common.a
COMMAND ar -x ${THIRDPARTY_INSTALL_DIR}/lib/libthrift.a
COMMAND ar -x ${CMAKE_BINARY_DIR}/src/base/libpegasus_base.a
COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/$<TARGET_NAME:pegasus_client_impl_objects>.dir/*.o .
COMMAND ar -qcs ${pegasus_client_static_lib} *.o
COMMAND rm -rf *.o
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
- DEPENDS pegasus_base pegasus_client_impl_objects
+ DEPENDS pegasus_base pegasus_client_impl_objects dsn_runtime dsn_client dsn_replication_common
COMMENT "generate pegasus client static library"
)
add_library(pegasus_client_static STATIC IMPORTED GLOBAL)
diff --git a/src/geo/test/CMakeLists.txt b/src/geo/test/CMakeLists.txt
index fbb17cd4b..b59934773 100644
--- a/src/geo/test/CMakeLists.txt
+++ b/src/geo/test/CMakeLists.txt
@@ -39,6 +39,6 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
add_definitions(-Wno-attributes)
-set(MY_BINPLACES "config.ini")
+set(MY_BINPLACES config.ini run.sh)
dsn_add_test()
diff --git a/src/geo/test/config.ini b/src/geo/test/config.ini
index 7c0337297..acc3fbc2a 100644
--- a/src/geo/test/config.ini
+++ b/src/geo/test/config.ini
@@ -24,7 +24,7 @@ count = 1
[apps.mimic]
type = dsn.app.mimic
arguments =
-pools = THREAD_POOL_DEFAULT
+pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER
run = true
count = 1
diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp
index 2a6426761..d4594772f 100644
--- a/src/geo/test/geo_test.cpp
+++ b/src/geo/test/geo_test.cpp
@@ -26,6 +26,10 @@
#include <dsn/utility/strings.h>
#include <dsn/utility/string_conv.h>
#include <base/pegasus_key_schema.h>
+#include <dsn/dist/fmt_logging.h>
+#include <dsn/dist/replication/replication_other_types.h>
+#include <dsn/dist/replication/replication_ddl_client.h>
+#include "base/pegasus_const.h"
namespace pegasus {
namespace geo {
@@ -35,6 +39,13 @@ class geo_client_test : public ::testing::Test
public:
geo_client_test()
{
+ std::vector<dsn::rpc_address> meta_list;
+ bool ok = dsn::replication::replica_helper::load_meta_servers(
+ meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), "onebox");
+ dassert_f(ok, "load_meta_servers failed");
+ auto ddl_client = new dsn::replication::replication_ddl_client(meta_list);
+ dsn::error_code error = ddl_client->create_app("temp_geo", "pegasus", 4, 3, {}, false);
+ dcheck_eq(dsn::ERR_OK, error);
_geo_client.reset(new pegasus::geo::geo_client("config.ini", "onebox", "temp", "temp_geo"));
}
diff --git a/src/geo/test/main.cpp b/src/geo/test/main.cpp
index dca522f87..8729cde46 100644
--- a/src/geo/test/main.cpp
+++ b/src/geo/test/main.cpp
@@ -23,6 +23,7 @@
GTEST_API_ int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
+ dsn_run_config("config.ini", false);
int ret = RUN_ALL_TESTS();
dsn_exit(ret);
}
diff --git a/src/geo/test/CMakeLists.txt b/src/geo/test/run.sh
old mode 100644
new mode 100755
similarity index 52%
copy from src/geo/test/CMakeLists.txt
copy to src/geo/test/run.sh
index fbb17cd4b..9090a865d
--- a/src/geo/test/CMakeLists.txt
+++ b/src/geo/test/run.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,9 +6,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -15,30 +16,13 @@
# specific language governing permissions and limitations
# under the License.
-set(MY_PROJ_NAME pegasus_geo_test)
-project(${MY_PROJ_NAME} C CXX)
+exit_if_fail() {
+ if [ $1 != 0 ]; then
+ echo $2
+ exit 1
+ fi
+}
-# Source files under CURRENT project directory will be automatically included.
-# You can manually set MY_PROJ_SRC to include source files under other directories.
-set(MY_PROJ_SRC "")
+./pegasus_geo_test
-# Search mode for source files under CURRENT project directory?
-# "GLOB_RECURSE" for recursive search
-# "GLOB" for non-recursive search
-set(MY_SRC_SEARCH_MODE "GLOB")
-
-set(MY_PROJ_LIBS
- pegasus_geo_lib
- s2testing
- s2
- pegasus_client_static
- dsn_utils
- gtest)
-
-set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
-
-add_definitions(-Wno-attributes)
-
-set(MY_BINPLACES "config.ini")
-
-dsn_add_test()
+exit_if_fail $? "run pegasus_geo_test failed"
diff --git a/src/rdsn/CMakeLists.txt b/src/rdsn/CMakeLists.txt
deleted file mode 100644
index b008a0a92..000000000
--- a/src/rdsn/CMakeLists.txt
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-project(dsn C CXX)
-
-set(RDSN_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
-
-# remove?
-set(DSN_BUILD_RUNTIME TRUE)
-
-dsn_common_setup()
-
-# Users don't have to configure CMAKE_INSTALL_PREFIX unless they want to customize
-# the destination.
-set(CMAKE_INSTALL_PREFIX ${DSN_ROOT} CACHE STRING "" FORCE)
-message (STATUS "dsn Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX})
-
-# install rdsn/include to ${CMAKE_INCLUDE_PREFIX}/include
-install(DIRECTORY include/ DESTINATION include)
-
-# NOTE: dsn_types.h is not actually a thrift-generated file. As pegasus may reference it,
-# we should copy also this file to the path the same as other thrift-generated files.
-install(FILES include/dsn/cpp/serialization_helper/dsn_types.h DESTINATION include)
-# TODO(wutao1): remove this line after dsn.layer2_types.h is removed from repo.
-install(FILES include/dsn/cpp/serialization_helper/dsn.layer2_types.h DESTINATION include)
-
-include_directories(${RDSN_PROJECT_ROOT}/include)
-include_directories(${RDSN_PROJECT_ROOT}/include/dsn/cpp/serialization_helper)
-include_directories(${RDSN_PROJECT_ROOT}/src)
-
-add_subdirectory(src)
diff --git a/src/rdsn/src/CMakeLists.txt b/src/rdsn/src/CMakeLists.txt
index 0ddbbfae0..7ef4c29d1 100644
--- a/src/rdsn/src/CMakeLists.txt
+++ b/src/rdsn/src/CMakeLists.txt
@@ -1,3 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+get_filename_component(RDSN_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
+message (STATUS "RDSN_PROJECT_ROOT = " ${RDSN_PROJECT_ROOT})
+
+# remove?
+set(DSN_BUILD_RUNTIME TRUE)
+
+dsn_common_setup()
+
+include_directories(${RDSN_PROJECT_ROOT}/include)
+include_directories(${RDSN_PROJECT_ROOT}/include/dsn/cpp/serialization_helper)
+include_directories(${RDSN_PROJECT_ROOT}/src)
+
if(UNIX)
add_compile_options(-fPIC)
endif()
diff --git a/src/rdsn/src/meta/CMakeLists.txt b/src/rdsn/src/meta/CMakeLists.txt
index 2e1e582ca..268bfffff 100644
--- a/src/rdsn/src/meta/CMakeLists.txt
+++ b/src/rdsn/src/meta/CMakeLists.txt
@@ -36,6 +36,8 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
# Extra files that will be installed
set(MY_BINPLACES "")
+add_definitions(-DDSN_MOCK_TEST)
+
dsn_add_shared_library()
add_subdirectory(test)
diff --git a/src/rdsn/src/meta/meta_service.cpp b/src/rdsn/src/meta/meta_service.cpp
index 6203cdcc8..39712eb6a 100644
--- a/src/rdsn/src/meta/meta_service.cpp
+++ b/src/rdsn/src/meta/meta_service.cpp
@@ -42,12 +42,10 @@
#include "meta_service.h"
#include "server_state.h"
-#include "meta_server_failure_detector.h"
#include "server_load_balancer.h"
#include "meta/duplication/meta_duplication_service.h"
#include "meta_split_service.h"
#include "meta_bulk_load_service.h"
-#include "runtime/security/access_controller.h"
namespace dsn {
namespace replication {
@@ -115,87 +113,6 @@ bool meta_service::check_freeze() const
return _alive_set.size() * 100 < _node_live_percentage_threshold_for_update * total;
}
-template <typename TRpcHolder>
-int meta_service::check_leader(TRpcHolder rpc, rpc_address *forward_address)
-{
- dsn::rpc_address leader;
- if (!_failure_detector->get_leader(&leader)) {
- if (!rpc.dsn_request()->header->context.u.is_forward_supported) {
- if (forward_address != nullptr)
- *forward_address = leader;
- return -1;
- }
-
- dinfo("leader address: %s", leader.to_string());
- if (!leader.is_invalid()) {
- rpc.forward(leader);
- return 0;
- } else {
- if (forward_address != nullptr)
- forward_address->set_invalid();
- return -1;
- }
- }
- return 1;
-}
-
-template <typename TRpcHolder>
-bool meta_service::check_status(TRpcHolder rpc, rpc_address *forward_address)
-{
- if (!_access_controller->allowed(rpc.dsn_request())) {
- rpc.response().err = ERR_ACL_DENY;
- ddebug("reject request with ERR_ACL_DENY");
- return false;
- }
-
- int result = check_leader(rpc, forward_address);
- if (result == 0)
- return false;
- if (result == -1 || !_started) {
- if (result == -1) {
- rpc.response().err = ERR_FORWARD_TO_OTHERS;
- } else if (_recovering) {
- rpc.response().err = ERR_UNDER_RECOVERY;
- } else {
- rpc.response().err = ERR_SERVICE_NOT_ACTIVE;
- }
- ddebug("reject request with %s", rpc.response().err.to_string());
- return false;
- }
-
- return true;
-}
-
-template <typename TRespType>
-bool meta_service::check_status_with_msg(message_ex *req, TRespType &response_struct)
-{
- if (!_access_controller->allowed(req)) {
- ddebug("reject request with ERR_ACL_DENY");
- response_struct.err = ERR_ACL_DENY;
- reply(req, response_struct);
- return false;
- }
-
- int result = check_leader(req, nullptr);
- if (result == 0) {
- return false;
- }
- if (result == -1 || !_started) {
- if (result == -1) {
- response_struct.err = ERR_FORWARD_TO_OTHERS;
- } else if (_recovering) {
- response_struct.err = ERR_UNDER_RECOVERY;
- } else {
- response_struct.err = ERR_SERVICE_NOT_ACTIVE;
- }
- ddebug("reject request with %s", response_struct.err.to_string());
- reply(req, response_struct);
- return false;
- }
-
- return true;
-}
-
error_code meta_service::remote_storage_initialize()
{
// create storage
diff --git a/src/rdsn/src/meta/meta_service.h b/src/rdsn/src/meta/meta_service.h
index a71775fd2..b8b50967e 100644
--- a/src/rdsn/src/meta/meta_service.h
+++ b/src/rdsn/src/meta/meta_service.h
@@ -51,6 +51,8 @@
#include "meta_state_service_utils.h"
#include "block_service/block_service_manager.h"
#include "partition_guardian.h"
+#include "meta_server_failure_detector.h"
+#include "runtime/security/access_controller.h"
namespace dsn {
namespace security {
@@ -339,5 +341,86 @@ private:
std::atomic<meta_op_status> _meta_op_status;
};
+template <typename TRpcHolder>
+int meta_service::check_leader(TRpcHolder rpc, rpc_address *forward_address)
+{
+ dsn::rpc_address leader;
+ if (!_failure_detector->get_leader(&leader)) {
+ if (!rpc.dsn_request()->header->context.u.is_forward_supported) {
+ if (forward_address != nullptr)
+ *forward_address = leader;
+ return -1;
+ }
+
+ dinfo("leader address: %s", leader.to_string());
+ if (!leader.is_invalid()) {
+ rpc.forward(leader);
+ return 0;
+ } else {
+ if (forward_address != nullptr)
+ forward_address->set_invalid();
+ return -1;
+ }
+ }
+ return 1;
+}
+
+template <typename TRpcHolder>
+bool meta_service::check_status(TRpcHolder rpc, rpc_address *forward_address)
+{
+ if (!_access_controller->allowed(rpc.dsn_request())) {
+ rpc.response().err = ERR_ACL_DENY;
+ ddebug("reject request with ERR_ACL_DENY");
+ return false;
+ }
+
+ int result = check_leader(rpc, forward_address);
+ if (result == 0)
+ return false;
+ if (result == -1 || !_started) {
+ if (result == -1) {
+ rpc.response().err = ERR_FORWARD_TO_OTHERS;
+ } else if (_recovering) {
+ rpc.response().err = ERR_UNDER_RECOVERY;
+ } else {
+ rpc.response().err = ERR_SERVICE_NOT_ACTIVE;
+ }
+ ddebug("reject request with %s", rpc.response().err.to_string());
+ return false;
+ }
+
+ return true;
+}
+
+template <typename TRespType>
+bool meta_service::check_status_with_msg(message_ex *req, TRespType &response_struct)
+{
+ if (!_access_controller->allowed(req)) {
+ ddebug("reject request with ERR_ACL_DENY");
+ response_struct.err = ERR_ACL_DENY;
+ reply(req, response_struct);
+ return false;
+ }
+
+ int result = check_leader(req, nullptr);
+ if (result == 0) {
+ return false;
+ }
+ if (result == -1 || !_started) {
+ if (result == -1) {
+ response_struct.err = ERR_FORWARD_TO_OTHERS;
+ } else if (_recovering) {
+ response_struct.err = ERR_UNDER_RECOVERY;
+ } else {
+ response_struct.err = ERR_SERVICE_NOT_ACTIVE;
+ }
+ ddebug("reject request with %s", response_struct.err.to_string());
+ reply(req, response_struct);
+ return false;
+ }
+
+ return true;
+}
+
} // namespace replication
} // namespace dsn
diff --git a/src/rdsn/src/meta/test/CMakeLists.txt b/src/rdsn/src/meta/test/CMakeLists.txt
index 14c4fde26..e52786828 100644
--- a/src/rdsn/src/meta/test/CMakeLists.txt
+++ b/src/rdsn/src/meta/test/CMakeLists.txt
@@ -23,6 +23,7 @@ set(MY_PROJ_LIBS
dsn.failure_detector
dsn_dist_cmd
dsn_http
+ dsn_meta_server
dsn_runtime
dsn_aio
zookeeper_mt
@@ -40,7 +41,6 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
# Extra files that will be installed
set(MY_BINPLACES clear.sh run.sh config-test.ini suite1 suite2)
-add_definitions(-DDSN_MOCK_TEST)
dsn_add_test()
add_subdirectory(balancer_simulator)
diff --git a/src/rdsn/src/meta/test/run.sh b/src/rdsn/src/meta/test/run.sh
index 2c1b7e5c7..ad104aa6b 100755
--- a/src/rdsn/src/meta/test/run.sh
+++ b/src/rdsn/src/meta/test/run.sh
@@ -31,3 +31,17 @@ fi
./clear.sh
output_xml="${REPORT_DIR}/dsn.meta.test.1.xml"
GTEST_OUTPUT="xml:${output_xml}" ./dsn.meta.test
+if [ $? -ne 0 ]; then
+ echo "run dsn.meta.test failed"
+ echo "---- ls ----"
+ ls -l
+ if find . -name log.1.txt; then
+ echo "---- tail -n 100 log.1.txt ----"
+ tail -n 100 `find . -name log.1.txt`
+ fi
+ if [ -f core ]; then
+ echo "---- gdb ./dsn.meta.test core ----"
+ gdb ./dsn.meta.test core -ex "thread apply all bt" -ex "set pagination 0" -batch
+ fi
+ exit 1
+fi
diff --git a/src/rdsn/src/replica/test/replica_test.cpp b/src/rdsn/src/replica/test/replica_test.cpp
index 23841613b..5928128cc 100644
--- a/src/rdsn/src/replica/test/replica_test.cpp
+++ b/src/rdsn/src/replica/test/replica_test.cpp
@@ -237,11 +237,10 @@ private:
TEST_F(replica_test, write_size_limited)
{
int count = 100;
- task_code default_code;
struct dsn::message_header header;
header.body_length = 10000000;
- auto write_request = dsn::message_ex::create_request(default_code);
+ auto write_request = dsn::message_ex::create_request(RPC_TEST);
auto cleanup = dsn::defer([=]() { delete write_request; });
write_request->header = &header;
std::unique_ptr<tools::sim_network_provider> sim_net(
diff --git a/src/rdsn/src/runtime/rpc/rpc_message.cpp b/src/rdsn/src/runtime/rpc/rpc_message.cpp
index 664e450c6..8425c60c3 100644
--- a/src/rdsn/src/runtime/rpc/rpc_message.cpp
+++ b/src/rdsn/src/runtime/rpc/rpc_message.cpp
@@ -289,7 +289,7 @@ message_ex *message_ex::create_request(dsn::task_code rpc_code,
// init header
auto &hdr = *msg->header;
memset(static_cast<void *>(&hdr), 0, sizeof(hdr));
- hdr.hdr_type = *(uint32_t *)"RDSN";
+ hdr.hdr_type = 0x4e534452; // "RDSN"
hdr.hdr_length = sizeof(message_header);
hdr.hdr_crc32 = hdr.body_crc32 = CRC_INVALID;
diff --git a/src/rdsn/src/runtime/rpc/thrift_message_parser.h b/src/rdsn/src/runtime/rpc/thrift_message_parser.h
index 7fa560389..8bf268206 100644
--- a/src/rdsn/src/runtime/rpc/thrift_message_parser.h
+++ b/src/rdsn/src/runtime/rpc/thrift_message_parser.h
@@ -79,7 +79,7 @@ struct v1_specific_vars
std::unique_ptr<thrift_request_meta_v1> _meta_v1;
};
-#define THRIFT_HDR_SIG (*(uint32_t *)"THFT")
+static const uint32_t THRIFT_HDR_SIG = 0x54464854; //"THFT"
DEFINE_CUSTOMIZED_ID(network_header_format, NET_HDR_THRIFT)
diff --git a/src/rdsn/src/runtime/task/task_spec.cpp b/src/rdsn/src/runtime/task/task_spec.cpp
index c7fcd30a1..6b469ec86 100644
--- a/src/rdsn/src/runtime/task/task_spec.cpp
+++ b/src/rdsn/src/runtime/task/task_spec.cpp
@@ -35,6 +35,7 @@
#include <array>
+#include <dsn/dist/fmt_logging.h>
#include <dsn/tool-api/task_spec.h>
#include <dsn/tool-api/command_manager.h>
#include <dsn/tool-api/threadpool_spec.h>
@@ -58,7 +59,8 @@ void task_spec::register_task_code(task_code code,
dsn_task_priority_t pri,
dsn::threadpool_code pool)
{
- dassert(code < TASK_SPEC_STORE_CAPACITY, "code = %d", code);
+ dcheck_ge(code, 0);
+ dcheck_lt(code, TASK_SPEC_STORE_CAPACITY);
if (!s_task_spec_store[code]) {
s_task_spec_store[code] = make_unique<task_spec>(code, code.to_string(), type, pri, pool);
auto &spec = s_task_spec_store[code];
@@ -127,7 +129,9 @@ void task_spec::register_storage_task_code(task_code code,
task_spec *task_spec::get(int code)
{
- return code < TASK_SPEC_STORE_CAPACITY ? s_task_spec_store[code].get() : nullptr;
+ dcheck_ge(code, 0);
+ dcheck_lt(code, TASK_SPEC_STORE_CAPACITY);
+ return s_task_spec_store[code].get();
}
task_spec::task_spec(int code,
diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp
index d7e5d2b7f..d05bd7bdc 100644
--- a/src/server/hotkey_collector.cpp
+++ b/src/server/hotkey_collector.cpp
@@ -84,6 +84,10 @@ find_outlier_index(const std::vector<uint64_t> &captured_keys, int threshold, in
hot_value = captured_keys[i];
}
}
+ if (hot_index == -1) {
+ return false;
+ }
+
// TODO: (Tangyanzhao) increase a judgment of table_captured_key_sum
double captured_keys_avg_count =
(table_captured_key_sum - captured_keys[hot_index]) / (data_size - 1);
diff --git a/src/test/function_test/test_batch_get.cpp b/src/test/function_test/test_batch_get.cpp
index f44c95fd5..cb3778754 100644
--- a/src/test/function_test/test_batch_get.cpp
+++ b/src/test/function_test/test_batch_get.cpp
@@ -48,19 +48,20 @@ TEST(batch_get, set_and_then_batch_get)
uint64_t test_partition_hash = 0;
apps::batch_get_request batch_request;
- std::vector<std::pair<std::string, std::string>> key_pair_list;
- std::vector<std::string> value_list;
+ std::vector<std::string> test_data_hash_keys(test_data_count);
+ std::vector<std::string> test_data_sort_keys(test_data_count);
+ std::vector<std::string> test_data_values(test_data_count);
for (int i = 0; i < test_data_count; ++i) {
- std::string hash_key = "hash_key_prefix_" + std::to_string(i);
- std::string sort_key = "sort_key_prefix_" + std::to_string(i);
- std::string value = "value_" + std::to_string(i);
+ test_data_hash_keys[i] = "hash_key_prefix_" + std::to_string(i);
+ test_data_sort_keys[i] = "sort_key_prefix_" + std::to_string(i);
+ test_data_values[i] = "value_" + std::to_string(i);
apps::update_request one_request;
one_request.__isset.key = true;
- pegasus_generate_key(one_request.key, hash_key, sort_key);
+ pegasus_generate_key(one_request.key, test_data_hash_keys[i], test_data_sort_keys[i]);
one_request.__isset.value = true;
- one_request.value.assign(value.c_str(), 0, value.size());
+ one_request.value.assign(test_data_values[i].c_str(), 0, test_data_values[i].size());
auto put_result = rrdb_client->put_sync(
one_request, std::chrono::milliseconds(test_timeout_milliseconds), test_partition_hash);
ASSERT_EQ(ERR_OK, put_result.first);
@@ -68,25 +69,29 @@ TEST(batch_get, set_and_then_batch_get)
apps::full_key one_full_key;
one_full_key.__isset.hash_key = true;
- one_full_key.hash_key.assign(hash_key.c_str(), 0, hash_key.size());
+ one_full_key.hash_key.assign(
+ test_data_hash_keys[i].c_str(), 0, test_data_hash_keys[i].size());
one_full_key.__isset.sort_key = true;
- one_full_key.sort_key.assign(sort_key.c_str(), 0, sort_key.size());
+ one_full_key.sort_key.assign(
+ test_data_sort_keys[i].c_str(), 0, test_data_sort_keys[i].size());
batch_request.keys.emplace_back(std::move(one_full_key));
-
- key_pair_list.emplace_back(std::move(hash_key), std::move(sort_key));
- value_list.push_back(std::move(value));
}
int test_no_exist_data_count = 6;
+ std::vector<std::string> no_exist_data_hash_keys(test_no_exist_data_count);
+ std::vector<std::string> no_exist_data_sort_keys(test_no_exist_data_count);
+ std::vector<std::string> no_exist_data_values(test_no_exist_data_count);
for (int i = 0; i < test_no_exist_data_count; ++i) {
- std::string hash_key = "hash_key_prefix_no_exist_" + std::to_string(i);
- std::string sort_key = "sort_key_prefix_no_exist_" + std::to_string(i);
+ no_exist_data_hash_keys[i] = "hash_key_prefix_no_exist_" + std::to_string(i);
+ no_exist_data_sort_keys[i] = "sort_key_prefix_no_exist_" + std::to_string(i);
apps::full_key one_full_key;
one_full_key.__isset.hash_key = true;
- one_full_key.hash_key.assign(hash_key.c_str(), 0, hash_key.size());
+ one_full_key.hash_key.assign(
+ no_exist_data_hash_keys[i].c_str(), 0, no_exist_data_hash_keys[i].size());
one_full_key.__isset.sort_key = true;
- one_full_key.sort_key.assign(sort_key.c_str(), 0, sort_key.size());
+ one_full_key.sort_key.assign(
+ no_exist_data_sort_keys[i].c_str(), 0, no_exist_data_sort_keys[i].size());
batch_request.keys.emplace_back(std::move(one_full_key));
}
@@ -97,8 +102,8 @@ TEST(batch_get, set_and_then_batch_get)
ASSERT_EQ(rocksdb::Status::kOk, response.error);
ASSERT_EQ(test_data_count, response.data.size());
for (int i = 0; i < test_data_count; ++i) {
- ASSERT_EQ(response.data[i].hash_key.to_string(), key_pair_list[i].first);
- ASSERT_EQ(response.data[i].sort_key.to_string(), key_pair_list[i].second);
- ASSERT_EQ(response.data[i].value.to_string(), value_list[i]);
+ ASSERT_EQ(response.data[i].hash_key.to_string(), test_data_hash_keys[i]);
+ ASSERT_EQ(response.data[i].sort_key.to_string(), test_data_sort_keys[i]);
+ ASSERT_EQ(response.data[i].value.to_string(), test_data_values[i]);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org