diff --git a/.gitignore b/.gitignore index 538c92fc0f33829e234c32ec51f8dccacf96040b..315f68452603204db8d098e3322ade96cafe939f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,8 +32,8 @@ build.ninja ### build build/ -### thirdparty build cache -thirdparty/thirdparty/example/build +### resource +resource/ ### functionsystem build cache functionsystem/build/CMakeFiles diff --git a/common/metrics/metrics.cmake b/common/metrics/metrics.cmake index 9e82e9a9530b155d0b8f423cbd593a25582ea3cc..1f9bc707d67c3cb9d0fa2558a9d28cb0c2ed263d 100644 --- a/common/metrics/metrics.cmake +++ b/common/metrics/metrics.cmake @@ -31,7 +31,6 @@ set(metrics_sdk_LIB ${metrics_LIB_DIR}/libobservability-metrics-sdk.so) set(metrics_file_exporter_LIB ${metrics_LIB_DIR}/libobservability-metrics-file-exporter.so) set(metrics_exporter_ostream_LIB ${metrics_LIB_DIR}/libobservability-metrics-exporter-ostream.so) set(metrics_prometheus_push_exporter_LIB ${metrics_LIB_DIR}/libobservability-prometheus-push-exporter.so) -set(metrics_aom_alarm_exporter_LIB ${metrics_LIB_DIR}/libobservability-aom-alarm-exporter.so) include_directories(${metrics_INCLUDE_DIR}) @@ -42,6 +41,5 @@ install(FILES ${metrics_sdk_LIB} ${metrics_file_exporter_LIB} ${metrics_exporter_ostream_LIB} - ${metrics_aom_alarm_exporter_LIB} ${metrics_prometheus_push_exporter_LIB} DESTINATION lib) diff --git a/functionsystem/build.sh b/functionsystem/build.sh index f49b0ef5f3798a117ed4ba6083aa7d731322baab..ea391452cac3a0173f1300a39dbbd9f340911475 100644 --- a/functionsystem/build.sh +++ b/functionsystem/build.sh @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +set -x set -e set -o nounset set -o pipefail @@ -113,7 +113,7 @@ THIRDPARTY_INSTALL_DIR="${THIRDPARTY_SRC_DIR}/out" BUILD_FUNCTIONCORE="OFF" FUNCTIONCORE_SRC_DIR="${YR_ROOT_DIR}/functionsystem" FUNCTIONCORE_OUT_DIR="${YR_ROOT_DIR}/output/functioncore" -YUANRONG_DIR="${YR_ROOT_DIR}/../output/yuanrong" +YUANRONG_DIR="${YR_ROOT_DIR}/output/function_system" # go module prepare export GO111MODULE=on @@ -153,23 +153,9 @@ function check_module() { function check_posix() { log_info "Start check posix at ${POSIX_DIR}" - if [ ! -d "${POSIX_DIR}" ]; then - mkdir -p "${POSIX_DIR}" - fi - if [ ! -d "${YR_ROOT_DIR}/posix" ]; then - log_error "The posix project does not exist, please check it" - exit 1 - fi - if [ ! -f "${POSIX_DIR}/common.proto" ]|| \ - [ ! -f "${POSIX_DIR}/core_service.proto" ]|| \ - [ ! -f "${POSIX_DIR}/runtime_rpc.proto" ]|| \ - [ ! -f "${POSIX_DIR}/affinity.proto" ]|| \ - [ ! -f "${POSIX_DIR}/runtime_service.proto" ]; then - cp "${YR_ROOT_DIR}/posix/proto"/*.proto "${POSIX_DIR}" - log_info "Get posix success" - else - log_info "posix file is exist" - fi + mkdir -p ${POSIX_DIR} + cp -f ${YR_ROOT_DIR}/proto/inner/*.proto ${POSIX_DIR} + cp -f ${YR_ROOT_DIR}/proto/posix/*.proto ${POSIX_DIR} } function strip_symbols() { @@ -218,12 +204,6 @@ function functioncore_compile() { bash build_golang.sh linux || die "cli module build failed" CLI_NAME="yr" bash build_golang.sh linux || die "cli module build failed" - rm -f "${PACKAGE_OUTPUT_DIR}"/${YR_VERSION}.tar.gz - cd ${PACKAGE_OUTPUT_DIR} - tar -czf "${PACKAGE_OUTPUT_DIR}"/${YR_VERSION}.tar.gz function_system - rm -rf function_system - cd ${YR_ROOT_DIR} - log_info "functioncore build successfully" } @@ -317,78 +297,8 @@ log_info "Begin to build, Build-Type:${BUILD_TYPE} Enable-LLT:${BUILD_LLT} Sanit if [ X"${BUILD_FUNCTIONCORE}" == X"ON" ]; then functioncore_compile - exit 0 -fi - -function clear_object_posix() { - local pb_object="${PROJECT_DIR}/src/common/proto/pb" - [ -d "${pb_object}" ] && rm -f "${pb_object}"/*.pb.* - - local posix_objext="${pb_object}/posix" - [ -d "${posix_objext}" ] && rm -rf "${posix_objext}" -} - -if [ "X${CLEAR_OUTPUT}" = "XON" ]; then - [ -d "${BUILD_DIR}" ] && rm -rf "${BUILD_DIR}" - [ -d "${OUTPUT_DIR}" ] && rm -rf "${OUTPUT_DIR}" - [ -d "${THIRDPARTY_SRC_DIR}" ] && rm -rf "${THIRDPARTY_SRC_DIR}" - [ -d "${THIRDPARTY_INSTALL_DIR}" ] && rm -rf "${THIRDPARTY_INSTALL_DIR}" - - clear_object_posix -fi - -# Check and get Posix -check_posix - -# Build and install -echo cmake -G Ninja "${PROJECT_DIR}" -DCMAKE_INSTALL_PREFIX="${OUTPUT_DIR}" \ - -DBUILD_VERSION="${YR_VERSION}" \ - -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -DSANITIZERS="${SANITIZERS}" \ - -DBUILD_LLT="${BUILD_LLT}" \ - -DBUILD_GCOV="${BUILD_GCOV}" \ - -DBUILD_THREAD_NUM="${JOB_NUM}" \ - -DROOT_DIR="${YR_ROOT_DIR}" \ - -DJEMALLOC_PROF_ENABLE="${JEMALLOC_PROF_ENABLE}" \ - -DFUNCTION_SYSTEM_BUILD_TIME_TRACE="${FUNCTION_SYSTEM_BUILD_TIME_TRACE}" \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -mkdir -p "${BUILD_DIR}" && cd "${BUILD_DIR}"/ -cmake -G Ninja "${PROJECT_DIR}" -DCMAKE_INSTALL_PREFIX="${OUTPUT_DIR}" \ - -DBUILD_VERSION="${YR_VERSION}" \ - -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -DSANITIZERS="${SANITIZERS}" \ - -DBUILD_LLT="${BUILD_LLT}" \ - -DBUILD_GCOV="${BUILD_GCOV}" \ - -DBUILD_THREAD_NUM="${JOB_NUM}" \ - -DROOT_DIR="${YR_ROOT_DIR}" \ - -DJEMALLOC_PROF_ENABLE="${JEMALLOC_PROF_ENABLE}" \ - -DFUNCTION_SYSTEM_BUILD_TIME_TRACE="${FUNCTION_SYSTEM_BUILD_TIME_TRACE}" \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -echo "cmake configure successfully" - -# Compatible with EulerOS and Ubuntu -if [ ! -d "${OUTPUT_DIR}"/lib64 ]; then - mkdir -p "${OUTPUT_DIR}"/lib64 -fi - -if [ ! -d "${OUTPUT_DIR}"/lib ]; then - mkdir -p "${OUTPUT_DIR}"/lib fi -# If LLT is enabled to build, need to set LD_LIBRARY_PATH to link the generated shared library. -if [ "X${BUILD_LLT}" = "XON" ]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}:${OUTPUT_DIR}/lib:${OUTPUT_DIR}/lib64" - export BIN_PATH="${OUTPUT_DIR}/bin" # Integration test need this path to find test executables. - export NOT_SKIP_LONG_TESTS=0 # skip long test case in CI -fi - -echo "Start to compile ${MODULE} module" -# 编译指定组件,同时可以附加参数 -ninja ${MODULE} ${VERBOSE} -j "${JOB_NUM}" || die "Failed to compile ${MODULE}" -echo "ninja compile successfully" -cmake --build ${BUILD_DIR} --target install || die "Failed to install ${MODULE}" -echo "cmake install successfully" if [ "$BUILD_TYPE" != "Debug" ]; then [ ! -d "${OUTPUT_DIR}/functionsystem_SYM" ] && mkdir -p "${OUTPUT_DIR}/functionsystem_SYM" @@ -397,80 +307,39 @@ if [ "$BUILD_TYPE" != "Debug" ]; then strip_symbols "$OUTPUT_DIR"/lib "${OUTPUT_DIR}/functionsystem_SYM" fi -if command -v ccache &> /dev/null -then - ccache -s -fi - -# Tests -if [ "X${RUN_LLT}" = "XON" ]; then - pushd "bin" - - log_info "Run ut/it test units by python" - PY_PATH=${YR_ROOT_DIR}/scripts/executor/run_code_gate.py - IT_BIN_PATH=${BASE_DIR}/build/bin/${IT_EXECUTABLE} - UT_BIN_PATH=${BASE_DIR}/build/bin/${UT_EXECUTABLE} - python3 ${PY_PATH} --it_bin ${IT_BIN_PATH} --ut_bin ${UT_BIN_PATH} --test_suite "${TEST_SUIT}" --test_case "${TEST_CASE}" - EXIT_CODE=$? - echo "Run run_test_unit.py exit with code $EXIT_CODE" - if [ ${EXIT_CODE} -ne 0 ]; then - echo "Run run_test_unit.py with filter ${TEST_SUIT}.${TEST_CASE} filed." - exit 1 - fi - - popd -fi - -if [ "X${GEN_LLT_REPORT}" = "XON" ]; then - log_info "Generate test report" - coverage_file=coverage.info - find ./src -type f -name "*.gcda" -printf '%h\n' | sort | uniq | xargs -P ${JOB_NUM} -I {} sh -c 'lcov -c -d "{}" -o coverage_$(echo "{}" | sed "s/\//_/g").part_tmp' - lcov_files=$(find . -name "coverage_*.part_tmp" -size +0) - lcov_files_with_a=$(echo "$lcov_files" | xargs -n 1 echo -a) - lcov $lcov_files_with_a -o ${coverage_file}_tmp - - shielded_files=("*/common/utils/metadata/metadata.h" - "*/common/scheduler_framework/plugins/v1/preallocated_context.h" "*/common/utils/path.h" - "*/common/utils/files.h" "*/common/utils/actor_driver.h" - "*/common/utils/generate_message.h" "*/common/utils/hex.h" - "*/common/utils/param_check.h" "*/common/utils/proc_fs_tools.h" "*/common/utils/raii.h" - "*/common/utils/ssl_config.h" "*/common/utils/struct_transfer.h" "*/common/utils/ecdh_generator.cpp" - "*/common/utils/exec_utils.h" "*/common/utils/capability.h" "*/common/utils/cmd_tool.h" - "*/common/schedule_decision/queue/queue_item.h" "*/common/leader/leader_actor.h" - "*/common/utils/actor_worker.h" "*/common/meta_store/client/cpp/include/meta_store_client/txn_transaction.h" - "*/common/resource_view/resource_tool.h" "*/common/resource_view/scala_resource_tool.h" - "*/function_proxy/local_scheduler/local_sched_driver.cpp" - "*/function_proxy/busproxy/instance_proxy/perf.h" - "*/function_agent/code_deployer/obs_wrapper.h" - "*/common/scheduler_framework/utils/label_affinity_selector.h" - "*/common/scheduler_framework/framework/policy.h" - "*/output/include/metrics/*") # Coverage needs to be supplemented later. - lcov -r ${coverage_file}_tmp "*vendor*" "*logs*" "*.3rd*" "*usr*" "*.pb.*" "*/build/*" "*litebus*" "metrics" "*datasystem/output/sdk/cpp/include/*" \ - ${shielded_files[@]} -o ${coverage_file} - genhtml ${coverage_file} -o "coverage_report" - rm -f $(find . -name "coverage_*.part_tmp") -fi - # copy function system output -mkdir -p "${FUNCTION_SYSTEM_PACKAGE_DIR}"/bin "${FUNCTION_SYSTEM_PACKAGE_DIR}"/lib "${FUNCTION_SYSTEM_PACKAGE_DIR}"/include "${FUNCTION_SYSTEM_PACKAGE_DIR}"/deploy "${FUNCTION_SYSTEM_PACKAGE_DIR}"/config +# TODO 全是脚本耦合,待优化 +# OUTPUT_DIR = functionsystem/output +# FUNCTION_SYSTEM_PACKAGE_DIR = ./output/function_system +mkdir -p "${FUNCTION_SYSTEM_PACKAGE_DIR}"/bin "${FUNCTION_SYSTEM_PACKAGE_DIR}"/lib "${FUNCTION_SYSTEM_PACKAGE_DIR}"/deploy "${FUNCTION_SYSTEM_PACKAGE_DIR}"/config cp -ar "$OUTPUT_DIR"/bin/* "${FUNCTION_SYSTEM_PACKAGE_DIR}"/bin -if [ $(ls -A "$OUTPUT_DIR"/lib | wc -w) -ne 0 ]; then - cp -ar "$OUTPUT_DIR"/lib/* "${FUNCTION_SYSTEM_PACKAGE_DIR}"/lib -fi - -if [ -f "$OUTPUT_DIR"/include ]; then - cp -ar "$OUTPUT_DIR"/include/* "${FUNCTION_SYSTEM_PACKAGE_DIR}"/include -fi +cp -ar "$OUTPUT_DIR"/lib/* "${FUNCTION_SYSTEM_PACKAGE_DIR}"/lib +# 复制部署文件 cp -ar "${YR_ROOT_DIR}"/scripts/deploy/function_system/* "${FUNCTION_SYSTEM_PACKAGE_DIR}"/deploy -# copy metrics config file +# 复制可观测配置 cp -ar "${YR_ROOT_DIR}"/scripts/config/metrics/metrics_config.json "${FUNCTION_SYSTEM_PACKAGE_DIR}"/config/ +# clean and create output dir +[[ -d "${YUANRONG_DIR}"/third_party ]] && rm -rf "${YUANRONG_DIR}"/third_party +mkdir -p "${YUANRONG_DIR}"/third_party/etcd +cp "$BUILD_DIR"/etcd/bin/etcd "${YUANRONG_DIR}"/third_party/etcd +cp "$BUILD_DIR"/etcd/bin/etcdctl "${YUANRONG_DIR}"/third_party/etcd +cp "${YR_ROOT_DIR}"/scripts/deploy/third_party/health_check.sh "${YUANRONG_DIR}"/third_party/ +cp "${YR_ROOT_DIR}"/scripts/deploy/third_party/install.sh "${YUANRONG_DIR}"/third_party/ +cp "${YR_ROOT_DIR}"/scripts/deploy/third_party/utils.sh "${YUANRONG_DIR}"/third_party/ +find "${YUANRONG_DIR}"/third_party/ -type f -exec chmod 550 {} \; + if [ "$BUILD_TYPE" != "Debug" ]; then rm -rf "${SYM_OUTPUT_DIR}/functionsystem_SYM" cp -ar "${OUTPUT_DIR}/functionsystem_SYM" "${SYM_OUTPUT_DIR}/" tar -czf "${PACKAGE_OUTPUT_DIR}"/sym.tar.gz -C "${SYM_OUTPUT_DIR}/" . rm -rf "${SYM_OUTPUT_DIR}/" fi + +rm -f "${PACKAGE_OUTPUT_DIR}"/${YR_VERSION}.tar.gz +cd ${PACKAGE_OUTPUT_DIR} +tar -czf "${PACKAGE_OUTPUT_DIR}"/${YR_VERSION}.tar.gz function_system +cd ${YR_ROOT_DIR} \ No newline at end of file diff --git a/functionsystem/build_golang.sh b/functionsystem/build_golang.sh index edb8406f5b86eb421c98924c7f336df41b940da9..ee5ebe3afbf71aa0220ba5efbc47ec84f60ae891 100644 --- a/functionsystem/build_golang.sh +++ b/functionsystem/build_golang.sh @@ -35,6 +35,7 @@ MODE='-buildmode=pie' function build_linux() { CC='gcc -fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2' go build -buildmode=pie -ldflags "${FLAGS}" -o \ build/_output/bin/${CLI_NAME} ../../cmd/cli/main.go + ls -l build/_output/bin/${CLI_NAME} } function build_windows() { diff --git a/functionsystem/src/README.md b/functionsystem/src/README.md deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/functionsystem/src/common/README.md b/functionsystem/src/common/README.md deleted file mode 100644 index b225a8eb26ea206553c4deec5df891185d83e13f..0000000000000000000000000000000000000000 --- a/functionsystem/src/common/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# Common - -## Status - -### 错误码定义原则 -- 错误码使用十进制定义,有效范围[0, 10000),其他范围预留 -- 分组件定义,通过错误码最高位区分组件,[0, 1000)属于公共错误码 -- 不同组件允许定义类似功能错误码,当错误相同时能定位到具体组件 -- 公共错误码在组件内传递,组件错误码在组件间传递 -- 三方库错误码信息附加到组件错误信息中传递 - -```cpp -enum CompCode : int { - COMMON = 0, - BUSPROXY = 1000, - FUNCTION_ACCESSOR = 2000, - END = 10000, -}; - -enum StatusCode : int { - SUCCESS = COMMON + 0, - FAILED = COMMON + 1, - LOG_CONFIG_ERROR = COMMON + 2, - PARAMETER_ERROR = COMMON + 3, - - // Busproxy error code, range [1000, 2000) - BP_DATASYSTEM_ERROR = BUSPROXY + 1, - - // FunctionAccessor error code, range [2000, 3000) -}; -``` - -### 用法 -```cpp -auto status = Status::OK(); -status.ToString(); // [code: 0, status: No error occurs] - -auto status = Status(StatusCode::FAILED, "detail error message"); -status.ToString(); // [code: 1, status: Common error code - // detail: [detail error message]] - -auto status = Status(StatusCode::FAILED); -status.AppendMessage("detail error message"); -status.ToString(); // [code: 1, status: Common error code - // detail: [detail error message]] - -auto status = Status(StatusCode::FAILED, __LINE__, __FILE__, "detail error message"); -status.ToString(); // [code: 1, status: Common error code - // Line of code :* - // File :* - // detail: [detail error message]] -``` - -## Profile -### 用法 -```cpp -void Function1() -{ - PROFILE_FUNCTION(); - for (int i = 0; i < 10000; i++) { - std::cout << i << std::endl; - } - { - PROFILE_SCOPE("InFunction1"); - for (int i = 0; i < 20000; i++) { - std::cout << i << std::endl; - } - } -} - -void Function2() -{ - PROFILE_FUNCTION(); - for (int i = 0; i < 10000; i++) { - std::cout << i << "\n"; - } -} - -void RunBenckmarks() -{ - PROFILE_FUNCTION(); - std::thread t([]() { Function1(); }); - Function2(); - t.join(); -} - -int main() -{ - PROFILE_BEGIN_SESSION("profile", "profile.json"); - RunBenckmarks(); - PROFILE_END_SESSION(); -} -``` diff --git a/posix/README.md b/posix/README.md deleted file mode 100644 index 530cda7d8ac0f871d577734a729b6df863be751c..0000000000000000000000000000000000000000 --- a/posix/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# posix - -posix文件存放 \ No newline at end of file diff --git a/functionsystem/src/common/proto/posix/affinity.proto b/proto/inner/affinity.proto similarity index 100% rename from functionsystem/src/common/proto/posix/affinity.proto rename to proto/inner/affinity.proto diff --git a/functionsystem/src/common/proto/posix/bus_adapter.proto b/proto/inner/bus_adapter.proto similarity index 100% rename from functionsystem/src/common/proto/posix/bus_adapter.proto rename to proto/inner/bus_adapter.proto diff --git a/functionsystem/src/common/proto/posix/bus_service.proto b/proto/inner/bus_service.proto similarity index 100% rename from functionsystem/src/common/proto/posix/bus_service.proto rename to proto/inner/bus_service.proto diff --git a/functionsystem/src/common/proto/posix/common.proto b/proto/inner/common.proto similarity index 100% rename from functionsystem/src/common/proto/posix/common.proto rename to proto/inner/common.proto diff --git a/functionsystem/src/common/proto/posix/core_service.proto b/proto/inner/core_service.proto similarity index 100% rename from functionsystem/src/common/proto/posix/core_service.proto rename to proto/inner/core_service.proto diff --git a/functionsystem/src/common/proto/posix/inner_service.proto b/proto/inner/inner_service.proto similarity index 100% rename from functionsystem/src/common/proto/posix/inner_service.proto rename to proto/inner/inner_service.proto diff --git a/functionsystem/src/common/proto/posix/log_service.proto b/proto/inner/log_service.proto similarity index 100% rename from functionsystem/src/common/proto/posix/log_service.proto rename to proto/inner/log_service.proto diff --git a/functionsystem/src/common/proto/posix/message.proto b/proto/inner/message.proto similarity index 100% rename from functionsystem/src/common/proto/posix/message.proto rename to proto/inner/message.proto diff --git a/functionsystem/src/common/proto/posix/resource.proto b/proto/inner/resource.proto similarity index 100% rename from functionsystem/src/common/proto/posix/resource.proto rename to proto/inner/resource.proto diff --git a/functionsystem/src/common/proto/posix/runtime_rpc.proto b/proto/inner/runtime_rpc.proto similarity index 100% rename from functionsystem/src/common/proto/posix/runtime_rpc.proto rename to proto/inner/runtime_rpc.proto diff --git a/functionsystem/src/common/proto/posix/runtime_service.proto b/proto/inner/runtime_service.proto similarity index 100% rename from functionsystem/src/common/proto/posix/runtime_service.proto rename to proto/inner/runtime_service.proto diff --git a/posix/proto/affinity.proto b/proto/posix/affinity.proto similarity index 100% rename from posix/proto/affinity.proto rename to proto/posix/affinity.proto diff --git a/posix/proto/common.proto b/proto/posix/common.proto similarity index 100% rename from posix/proto/common.proto rename to proto/posix/common.proto diff --git a/posix/proto/core_service.proto b/proto/posix/core_service.proto similarity index 100% rename from posix/proto/core_service.proto rename to proto/posix/core_service.proto diff --git a/posix/proto/runtime_rpc.proto b/proto/posix/runtime_rpc.proto similarity index 100% rename from posix/proto/runtime_rpc.proto rename to proto/posix/runtime_rpc.proto diff --git a/posix/proto/runtime_service.proto b/proto/posix/runtime_service.proto similarity index 100% rename from posix/proto/runtime_service.proto rename to proto/posix/runtime_service.proto diff --git a/scripts/basic_build.sh b/scripts/basic_build.sh index f9b2d31ec5710d9f27076169561e2b6429af1575..7eca42dc8f25ab42a1487bc54a1ed03df12ea4f0 100644 --- a/scripts/basic_build.sh +++ b/scripts/basic_build.sh @@ -21,7 +21,7 @@ BASE_DIR=$( ) YR_ROOT_DIR="${BASE_DIR}/../" YUANRONG_DIR="${YR_ROOT_DIR}/output/function_system" -BUILD_DIR="$YR_ROOT_DIR"/build +BUILD_DIR="$YR_ROOT_DIR"/functionsystem/build [[ ! -f "${YR_ROOT_DIR}/tools/utils.sh" ]] && echo "${YR_ROOT_DIR}/tools/utils.sh is not exist" && exit 1 . ${YR_ROOT_DIR}/tools/utils.sh @@ -61,16 +61,6 @@ function build_etcd() cd "$BUILD_DIR"/etcd/ bash build.sh - # clean and create output dir - [[ -d "${YUANRONG_DIR}"/third_party ]] && rm -rf "${YUANRONG_DIR}"/third_party - mkdir -p "${YUANRONG_DIR}"/third_party/etcd - cp "$BUILD_DIR"/etcd/bin/etcd "${YUANRONG_DIR}"/third_party/etcd - cp "$BUILD_DIR"/etcd/bin/etcdctl "${YUANRONG_DIR}"/third_party/etcd - cp "${BASE_DIR}"/deploy/third_party/health_check.sh "${YUANRONG_DIR}"/third_party/ - cp "${BASE_DIR}"/deploy/third_party/install.sh "${YUANRONG_DIR}"/third_party/ - cp "${BASE_DIR}"/deploy/third_party/utils.sh "${YUANRONG_DIR}"/third_party/ - find "${YUANRONG_DIR}"/third_party/ -type f -exec chmod 550 {} \; - log_info "---- build etcd success ----" } diff --git a/scripts/config/python_config/python-runtime-log.json b/scripts/config/python_config/python-runtime-log.json deleted file mode 100644 index 29a9c894805f0511b2b3990d1059e823aa802cef..0000000000000000000000000000000000000000 --- a/scripts/config/python_config/python-runtime-log.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "extra": { - "format": "[%(asctime)s %(levelname)s %(filename)s:%(lineno)d] [%(podname)s %(thread)d] %(message)s" - } - }, - "handlers": { - "file": { - "class": "logging.handlers.RotatingFileHandler", - "filename": "", - "formatter": "extra", - "maxBytes": 419430400, - "backupCount": 1 - } - }, - "loggers": { - "FileLogger": { - "handlers": [ - "file" - ], - "level": "{{logLevel}}" - } - } -} \ No newline at end of file diff --git a/scripts/config/python_config/runtime.json b/scripts/config/python_config/runtime.json deleted file mode 100644 index d8354f89ab6b4feac5931819db02a81cb42bda16..0000000000000000000000000000000000000000 --- a/scripts/config/python_config/runtime.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "maxRequestBodySize": "6", - "maxFdNum": 1024, - "dataSystemConnectionTimeout": "1", - "maxLogSize": 40, - "maxLogFileNum": 20 -} \ No newline at end of file diff --git a/scripts/deploy/data_system/health_check.sh b/scripts/deploy/data_system/health_check.sh deleted file mode 100644 index 5a7b020a3ba0ca7298f965f20a5cf4abe9764945..0000000000000000000000000000000000000000 --- a/scripts/deploy/data_system/health_check.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# must > 60 (waiting for datasystem start timeout) -datasystemTimeOut=65 -# max wait time (second) for data_system worker -readonly WAIT_DS_WORKER_MAX_TIME=$datasystemTimeOut -# max wait time (second) for data_system master -readonly WAIT_DS_MASTER_MAX_TIME=$datasystemTimeOut - -function health_check_by_pid_and_file() { - local pid=$1 - local file=$2 - local wait_max_time=$3 - local i=0 - local ret_code=0 - for ((i = 0; i <= wait_max_time; i++)); do - if ! kill -0 ${pid} &>/dev/null; then - ret_code=1 - return ${ret_code} - fi - if [ ! -f "${file}" ]; then - ret_code=2 - sleep 1 - continue - fi - return 0 - done - return ${ret_code} -} - -function data_system_health_check() { - case "$1" in - ds_worker) - health_check_by_pid_and_file "$2" "${DS_WORKER_HEALTH_CHECK_PATH}" ${WAIT_DS_WORKER_MAX_TIME} - ;; - ds_master) - health_check_by_pid_and_file "$2" "${DS_MASTER_HEALTH_CHECK_PATH}" ${WAIT_DS_MASTER_MAX_TIME} - ;; - *) - log_warning >&2 "Unknown component" - return 1 - ;; - esac - return $? -} diff --git a/scripts/deploy/data_system/install.sh b/scripts/deploy/data_system/install.sh deleted file mode 100644 index 85120851007b0f08005080a9d2ee78c753622c70..0000000000000000000000000000000000000000 --- a/scripts/deploy/data_system/install.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o pipefail - -DATA_SYSTEM_DEPLOY_DIR=$(dirname "$(readlink -f "$0")") -if [ -n "${BASE_DIR}" ]; then - DATA_SYSTEM_DEPLOY_DIR=${BASE_DIR}/../../data_system/deploy -fi -DATA_SYSTEM_DIR=$(readlink -m "${DATA_SYSTEM_DEPLOY_DIR}/..") -MAX_PROCESS_EXIT_TIMES=20 - -# before use functions in this script, ${BASE_DIR}/utils.sh should be imported first - -function install_ds_master() { - log_info "start data system master, port=${DS_MASTER_PORT}..." - local data_system_install_dir=${INSTALL_DIR_PARENT}/"ds_master_$$" # use ds_master_+pid as path, can't same as worker - - [[ ! -d "${data_system_install_dir}/rocksdb" ]] && mkdir -p "${data_system_install_dir}/rocksdb" - [[ ! -d "${data_system_install_dir}/socket" ]] && mkdir -p "${data_system_install_dir}/socket" - local ret_code=1 - DS_MASTER_HEALTH_CHECK_PATH="${data_system_install_dir}"/master/health - mkdir -p "${DS_LOG_PATH}/master" - if check_port "${DS_MASTER_IP}" "${DS_MASTER_PORT}"; then - OPENSSL_CONF="" LD_LIBRARY_PATH="${DATA_SYSTEM_DIR}/service/lib:${LD_LIBRARY_PATH}" "${DATA_SYSTEM_DIR}"/service/datasystem_worker -master_address="${DS_MASTER_IP}:${DS_MASTER_PORT}" \ - -log_dir="${DS_LOG_PATH}/master" -worker_address="${DS_MASTER_IP}:${DS_MASTER_PORT}" \ - -unix_domain_socket_dir="${data_system_install_dir}/socket" -v="${DS_DEBUG_LOG_LEVEL}" \ - -minloglevel="${DS_LOG_LEVEL}" \ - -health_check_path="${data_system_install_dir}"/master/health \ - -ready_check_path="${data_system_install_dir}"/master/ready \ - -enable_reconciliation=false \ - -rpc_thread_num=${DS_RPC_THREAD_NUM} \ - -heartbeat_interval_ms=${DS_HEARTBEAT_INTERVAL_MS} \ - -enable_multi_stubs=true \ - -max_log_size=${DS_LOG_ROLLING_MAX_SIZE} -max_log_file_num=${DS_LOG_ROLLING_MAX_FILES} \ - -node_dead_timeout_s=${DS_NODE_DEAD_TIMEOUT_S} \ - -node_timeout_s=${DS_NODE_TIMEOUT_S} \ - -backend_store_dir="${data_system_install_dir}/rocksdb" \ - -etcd_address="${ETCD_CLUSTER_ADDRESS}" \ - -other_az_names="${ETCD_TABLE_PREFIX}" \ - -etcd_target_name_override="${ETCD_TARGET_NAME_OVERRIDE}" \ - -enable_etcd_auth=${ENABLE_ETCD_AUTH} \ - -arena_per_tenant=${DS_ARENA_PER_TENANT} \ - -enable_fallocate=${DS_ENABLE_FALLOCATE} \ - -enable_huge_tlb=${DS_ENABLE_HUGE_TLB} \ - -enable_thp=${DS_ENABLE_THP} \ - -etcd_ca=${ETCD_SSL_BASE_PATH}/${ETCD_CA_FILE} \ - -etcd_cert=${ETCD_SSL_BASE_PATH}/${ETCD_CLIENT_CERT_FILE} \ - -etcd_key=${ETCD_SSL_BASE_PATH}/${ETCD_CLIENT_KEY_FILE} \ - -enable_component_auth=${DS_COMPONENT_AUTH_ENABLE} \ - -curve_key_dir=${CURVE_KEY_PATH} \ - -logfile_mode=416 \ - -l2_cache_type=${DS_L2_CACHE_TYPE} \ - -sfs_path=${DS_SFS_PATH} \ - -log_monitor=${DS_LOG_MONITOR_ENABLE} \ - -zmq_chunk_sz=${ZMQ_CHUNK_SZ} \ - -enable_lossless_data_exit_mode=${ENABLE_LOSSLESS_DATA_EXIT_MODE} \ - -enable_distributed_master=false -stderrthreshold=3 >> "${DS_LOG_PATH}"/ds_master${STD_LOG_SUFFIX} 2>&1 & - DS_MASTER_PID="$!" - if data_system_health_check "ds_master" "${DS_MASTER_PID}"; then - log_info "succeed to start data system master, port=${DS_MASTER_PORT}, pid=${DS_MASTER_PID}" - return 0 - else - if [ "$?" == 2 ];then - ret_code=2 - log_warning "data system master start timeout, killing data system master, pid=${DS_MASTER_PID}." - else - ret_code=98 - log_warning "data system master health_check failed, killing data system master, pid=${DS_MASTER_PID}." - fi - fi - kill -15 ${DS_MASTER_PID} - local j - for ((j = 0; j <= "$MAX_PROCESS_EXIT_TIMES"; j++)); do - if ! ps -ns ${DS_MASTER_PID} | grep ${DS_MASTER_PID} &>/dev/null; then - break - elif [ "$j" == "$MAX_PROCESS_EXIT_TIMES" ]; then - log_warning "data system master failed to exit, will be forced to exit." - kill -9 ${DS_MASTER_PID} - rm -rf "${DATASYSTEM_INSTALL_DIR}"/rocksdb - fi - sleep 0.5 - done - fi - return ${ret_code} -} - -function install_ds_worker() { - log_info "start data system worker, port=${DS_WORKER_PORT}..." - local data_system_install_path=${INSTALL_DIR_PARENT}/"ds_worker_$$" # use ds_master_+pid as path, can't same as master - - [[ ! -d "${data_system_install_path}/socket" ]] && mkdir -p "${data_system_install_path}/socket" - [[ ! -d "${data_system_install_path}/rocksdb" ]] && mkdir -p "${data_system_install_path}/rocksdb" - local spill_args="" - if [ "X$DS_SPILL_ENABLE" == "Xtrue" ]; then - [[ ! -d "$DS_SPILL_DIRECTORY" ]] && mkdir -p "$DS_SPILL_DIRECTORY" - DS_WORKER_SPILL_SIZE_LIMIT=$((DS_SPILL_SIZE_LIMIT * 1024 * 1024)) - spill_args="-spill_directory=$DS_SPILL_DIRECTORY/prefix -spill_size_limit=$DS_WORKER_SPILL_SIZE_LIMIT" - fi - DS_WORKER_HEALTH_CHECK_PATH="${data_system_install_path}"/worker/health - DS_MASTER_ADDRESS="${DS_MASTER_IP}:${DS_MASTER_PORT}" - if [ "X${ENABLE_DISTRIBUTED_MASTER}" = "Xtrue" ]; then - DS_MASTER_ADDRESS="" - fi - OPENSSL_CONF="" LD_LIBRARY_PATH="${DATA_SYSTEM_DIR}/service/lib:${LD_LIBRARY_PATH}" "${DATA_SYSTEM_DIR}"/service/datasystem_worker \ - -master_address=${DS_MASTER_ADDRESS} \ - -log_dir="${DS_LOG_PATH}" -shared_memory_size_mb=${MEM4DATA} -worker_address="${IP_ADDRESS}:${DS_WORKER_PORT}" \ - -unix_domain_socket_dir="${data_system_install_path}/socket" -v="${DS_DEBUG_LOG_LEVEL}" \ - -minloglevel="${DS_LOG_LEVEL}" \ - -health_check_path="${DS_WORKER_HEALTH_CHECK_PATH}" \ - -ready_check_path="${data_system_install_path}"/worker/ready \ - -enable_reconciliation=false \ - -rpc_thread_num=${DS_RPC_THREAD_NUM} \ - -client_dead_timeout_s=${DS_CLIENT_DEAD_TIMEOUT_S} \ - -heartbeat_interval_ms=${DS_HEARTBEAT_INTERVAL_MS} \ - -enable_multi_stubs=true \ - -max_log_size=${DS_LOG_ROLLING_MAX_SIZE} -max_log_file_num=${DS_LOG_ROLLING_MAX_FILES} \ - -max_client_num=${DS_MAX_CLIENT_NUM} $spill_args \ - -memory_reclamation_time_second=${DS_MEMORY_RECLAMATION_TIME_SECOND} \ - -node_dead_timeout_s=${DS_NODE_DEAD_TIMEOUT_S} \ - -node_timeout_s=${DS_NODE_TIMEOUT_S} \ - -backend_store_dir="${data_system_install_path}/rocksdb" \ - -arena_per_tenant=${DS_ARENA_PER_TENANT} \ - -enable_fallocate=${DS_ENABLE_FALLOCATE} \ - -enable_huge_tlb=${DS_ENABLE_HUGE_TLB} \ - -enable_thp=${DS_ENABLE_THP} \ - -etcd_address="${ETCD_CLUSTER_ADDRESS}" \ - -other_az_names="${ETCD_TABLE_PREFIX}" \ - -etcd_target_name_override="${ETCD_TARGET_NAME_OVERRIDE}" \ - -enable_etcd_auth=${ENABLE_ETCD_AUTH} \ - -etcd_ca=${ETCD_SSL_BASE_PATH}/${ETCD_CA_FILE} \ - -etcd_cert=${ETCD_SSL_BASE_PATH}/${ETCD_CLIENT_CERT_FILE} \ - -etcd_key=${ETCD_SSL_BASE_PATH}/${ETCD_CLIENT_KEY_FILE} \ - -enable_component_auth=${DS_COMPONENT_AUTH_ENABLE} \ - -curve_key_dir=${CURVE_KEY_PATH} -logfile_mode=416 \ - -l2_cache_type=${DS_L2_CACHE_TYPE} \ - -sfs_path=${DS_SFS_PATH} \ - -log_monitor=${DS_LOG_MONITOR_ENABLE} \ - -zmq_chunk_sz=${ZMQ_CHUNK_SZ} \ - -enable_lossless_data_exit_mode=${ENABLE_LOSSLESS_DATA_EXIT_MODE} \ - -enable_distributed_master=${ENABLE_DISTRIBUTED_MASTER} -stderrthreshold=3 >> "${DS_LOG_PATH}"/ds_worker${STD_LOG_SUFFIX} 2>&1 & - DS_WORKER_PID="$!" - log_info "succeed to start data system worker, port=${DS_WORKER_PORT}, pid=${DS_WORKER_PID}" -} - -function install_data_system() { - case "$1" in - ds_master) - install_ds_master - return $? - ;; - ds_worker) - install_ds_worker - return $? - ;; - *) - log_warning >&2 "Unknown component $1" - return 1 - ;; - esac -} diff --git a/scripts/executor/compile/compile_cpp.py b/scripts/executor/compile/compile_cpp.py index f84928148011ca70115bf09691b5460557ccf3b6..23b76f4a76f5ef01ab81a7e2af2c61f5fe783515 100644 --- a/scripts/executor/compile/compile_cpp.py +++ b/scripts/executor/compile/compile_cpp.py @@ -1,6 +1,7 @@ # coding=UTF-8 # Copyright (c) 2025 Huawei Technologies Co., Ltd import json +import shutil import os.path import utils @@ -12,16 +13,22 @@ def compile_gtest(root_dir, job_num): compile_functionsystem(root_dir, job_num, build_type="Debug", gtest=True) -def compile_binary(root_dir, job_num): - compile_functionsystem(root_dir, job_num, build_type="Release") +def compile_binary(root_dir, job_num, version): + compile_functionsystem(root_dir, job_num, build_type="Release", version=version) def compile_functionsystem(root_dir, job_num, version="0.0.0", build_type="Debug", time_trace=False, coverage=False, jemalloc=False, sanitizers=False, gtest=False): print("Build cpp code in functionsystem") - # 检查 Proto 文件存在性 - log.warning("Copy proto file not implement") + # 拷贝 proto 文件 + log.info("Auto copy all proto file to cpp common folder") + inner_proto = os.path.join(root_dir, "proto", "inner") + posix_proto = os.path.join(root_dir, "proto", "posix") + cpp_proto_dir = os.path.join(root_dir, "functionsystem", "src", "common", "proto", "posix") + os.makedirs(cpp_proto_dir, exist_ok=True) + copy_proto_folder(inner_proto, cpp_proto_dir) + copy_proto_folder(posix_proto, cpp_proto_dir) # 使用 CMake 创建 Ninja 构建清单 root_dir = os.path.abspath(root_dir) # Git根目录 @@ -77,3 +84,8 @@ def version_name(version): def bool2switch(b: bool): return "ON" if b else "OFF" + +def copy_proto_folder(src, dst): + for proto_file in os.listdir(src): + if proto_file.endswith(".proto"): + shutil.copy(os.path.join(src, proto_file), dst) \ No newline at end of file diff --git a/scripts/executor/tasks/build_task.py b/scripts/executor/tasks/build_task.py index 9aa6311b820bc3f11bd2ed535b55a83197d3b766..87f6931fdc17d37f2b230f3fafb7b2c5a6ee4dd3 100644 --- a/scripts/executor/tasks/build_task.py +++ b/scripts/executor/tasks/build_task.py @@ -1,8 +1,9 @@ # Copyright (c) 2025 Huawei Technologies Co., Ltd -import json import os +import json import utils import tasks +import compile log = utils.stream_logger() @@ -21,7 +22,7 @@ def run_build(root_dir, cmd_args): compile_logs(args) compile_litebus(args) compile_metrics(args) - compile_functionsystem(args) + compile_functionsystem(root_dir, args) def compile_vendor(args): @@ -71,13 +72,10 @@ def compile_metrics(args): ) -def compile_functionsystem(args): +def compile_functionsystem(root_dir, args): cwd = os.path.join(args['root_dir'], "functionsystem") log.info("Start to compile functionsystem") - utils.sync_command( - ["bash", "build.sh", "-r", "-j", str(args['job_num']), "-v", args['version']], - cwd=cwd - ) + compile.compile_binary(root_dir, args["job_num"], args["version"]) utils.sync_command( ["bash", "build.sh", "-y", "-j", str(args['job_num']), "-v", args['version']], cwd=cwd diff --git a/scripts/executor/tasks/test_task.py b/scripts/executor/tasks/test_task.py index 8ebc77c4157e74de7b0eb88e0cbf39e163dc7d2f..1591009c0907fb33a6ebe43b5853b4e86cd3df97 100644 --- a/scripts/executor/tasks/test_task.py +++ b/scripts/executor/tasks/test_task.py @@ -39,9 +39,12 @@ def run_test(root_dir, cmd_args): if args['action'] in ["all", "exec"]: # 执行测试用例 log.info(f"Step(2/3, action = [exec]): Exec test case with args: {json.dumps(args)}") - tasks.run_code_gate(args["it_bin"], args["ut_bin"], args["test_suite"], args["test_case"], + exit_code = tasks.run_code_gate(args["it_bin"], args["ut_bin"], args["test_suite"], args["test_case"], exec_timeout=args['exec_timeout'], retry_times=args['retry_times'], job_num=args['job_num'], print_logs=args['print_logs']) + if exit_code != 0: + log.error(f"Run the test case and exit the code with a non-zero value of {exit_code}. Program termination") + exit(exit_code) else: log.info(f"Step(2/3, action = [exec]): Skip to exec test case") diff --git a/scripts/executor/utils/process.py b/scripts/executor/utils/process.py index fd47a3ce100ae5db9b56a85483905985a5f5f71d..0468bd17dccbe42899e5830d8076f2546f0b8606 100644 --- a/scripts/executor/utils/process.py +++ b/scripts/executor/utils/process.py @@ -20,6 +20,7 @@ def sync_command(cmd: list[str], cwd: str = None): cmd, cwd=cwd, text=True, + check=True ) diff --git a/scripts/executor/utils/tools.py b/scripts/executor/utils/tools.py index 48bc52fe560c26b39e7d9a32c30db0b9e01e7ca5..646217cfc94f075861f9bf279b4ec90e2ee7ff40 100644 --- a/scripts/executor/utils/tools.py +++ b/scripts/executor/utils/tools.py @@ -43,4 +43,4 @@ def get_linux_distribution(): for line in lines: if line.startswith("ID="): return line.split('"')[1] - return "Unknown" \ No newline at end of file + return "Unknown"