From 9056bd6c8694c367e024e8c3711293644f33d99a Mon Sep 17 00:00:00 2001 From: WolfBolin Date: Mon, 10 Nov 2025 17:05:55 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E3=80=90LLT=E3=80=91=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E6=89=A7=E8=A1=8C=E5=B7=A5?= =?UTF-8?q?=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functionsystem/build.sh | 112 +------ functionsystem/tests/integration/main.cpp | 4 +- functionsystem/tests/unit/main.cpp | 3 +- scripts/executor/run_code_gate.py | 365 ++++++++++++++++++++++ scripts/executor/utils/__init__.py | 7 + scripts/executor/utils/tools.py | 83 +++++ 6 files changed, 470 insertions(+), 104 deletions(-) create mode 100644 scripts/executor/run_code_gate.py create mode 100644 scripts/executor/utils/__init__.py create mode 100644 scripts/executor/utils/tools.py diff --git a/functionsystem/build.sh b/functionsystem/build.sh index 0d82bb3..aff83ae 100644 --- a/functionsystem/build.sh +++ b/functionsystem/build.sh @@ -662,113 +662,21 @@ function run_unit_test() fi } -function generate_temp_services_yaml() -{ - cat << EOF > /tmp/services.yaml -- service: defaultservice # 服务名（必填） - kind: yrlib # 函数类型（必填） 进程部署只支持 yrlib - description: this is the default service # 函数描述（非必填） - functions: # 函数（必填） - default: # 函数名（必填） - cpu: 0 # 函数 CPU 大小，单位：1/1000 核（必填） - memory: 0 # 函数 MEM 大小，单位：MB（必填） - runtime: python3.11 # 函数 runtime 类型（必填） - # environment: # 函数环境变量，内置环境变量定义 - # "key1" : "value1" - # storageType: "local" # 代码包存储类型，默认值为 local, 进程部署只支持 local: 代码包存储在磁盘中 - # codePath: "/home/sn/function-packages" # 代码包本地路径，默认值为空（除了 python，此项必填） - py: - cpu: 0 - memory: 0 - runtime: python3.11 - java: - cpu: 500 - memory: 500 - runtime: java1.8 - cpp: - cpu: 500 - memory: 500 - runtime: cpp11 -EOF -} - # Tests if [ "X${RUN_LLT}" = "XON" ]; then pushd "bin" - log_info "Running ut/it tests" - rm -rf /tmp/services.yaml - generate_temp_services_yaml - cp ${BUILD_DIR}/lib/libyaml_tool.so /tmp/libyaml_tool.so - rm -rf /tmp/executor-meta/ - mkdir -p /tmp/executor-meta/ - - log_info "Running tests in parallel subShells..." - - set +e - ( run_integration_test ) > /tmp/integration_test.log 2>&1 & - PID1=$! - echo "Started integration_test with PID: $PID1" - if [ "X${TEST_SUIT}" = "X*" ] && [ "X${TEST_CASE}" = "X*" ]; then - ( run_unit_test ) > /tmp/unit_test.log 2>&1 & - else - ( run_unit_test_specify_case ) > /tmp/unit_test.log 2>&1 & + log_info "Run ut/it test units by python" + PY_PATH=${YR_ROOT_DIR}/scripts/executor/run_code_gate.py + IT_BIN_PATH=${BASE_DIR}/build/bin/${IT_EXECUTABLE} + UT_BIN_PATH=${BASE_DIR}/build/bin/${UT_EXECUTABLE} + python3 ${PY_PATH} --it_bin ${IT_BIN_PATH} --ut_bin ${UT_BIN_PATH} --test_suite "${TEST_SUIT}" --test_case "${TEST_CASE}" + EXIT_CODE=$? + echo "Run run_test_unit.py exit with code $EXIT_CODE" + if [ ${EXIT_CODE} -ne 0 ]; then + echo "Run run_test_unit.py with filter ${TEST_SUIT}.${TEST_CASE} filed." + exit 1 fi - PID2=$! - echo "Started unit_test with PID: $PID2" - - # Dynamically wait for any one of the processes to finish - wait -n $PID1 $PID2 - EXIT_CODE=$? # Capture the status code of the first process to exit - - # Determine which process exited first - if ! kill -0 $PID1 2>/dev/null; then - FAILED_PID=$PID1 - REMAINING_PID=$PID2 - LOG_FILE="/tmp/integration_test.log" - TEST_NAME="integration_test" - else - FAILED_PID=$PID2 - REMAINING_PID=$PID1 - LOG_FILE="/tmp/unit_test.log" - TEST_NAME="unit_test" - fi - - # If the first process to exit failed - if [ $EXIT_CODE -ne 0 ]; then - echo "$TEST_NAME failed, exit code: $EXIT_CODE" - #Immediately kill the other test that is still running - kill -g $REMAINING_PID 2>/dev/null - # Force wait to prevent zombie processes - wait $REMAINING_PID 2>/dev/null - # Output failure log - echo "<<<<<<<<<<< $TEST_NAME log" - cat $LOG_FILE - cat /tmp/failed_unit_test.log - echo "<<<<<<<<<<< End of $TEST_NAME log, $TEST_NAME fail first" - exit 1 - else - # If the first process to exit was successful, continue waiting for the other - wait $REMAINING_PID - EXIT_REMAIN=$? - if [ $EXIT_REMAIN -ne 0 ]; then - if [ $REMAINING_PID -eq $PID1 ]; then - echo "integration_test failed, exit code: $EXIT_REMAIN" - LOG_FILE="/tmp/integration_test.log" - else - echo "unit_test failed, exit code: $EXIT_REMAIN" - LOG_FILE="/tmp/unit_test.log" - fi - echo "<<<<<<<<<<< Failure log" - cat $LOG_FILE - cat /tmp/failed_unit_test.log - echo "<<<<<<<<<<< End of $LOG_FILE log" - exit 1 - else - echo "All tests passed" - fi - fi - set -e popd fi diff --git a/functionsystem/tests/integration/main.cpp b/functionsystem/tests/integration/main.cpp index 65b2924..b700d08 100644 --- a/functionsystem/tests/integration/main.cpp +++ b/functionsystem/tests/integration/main.cpp @@ -39,7 +39,9 @@ const std::string LOG_CONFIG_JSON = R"( "maxQueueSize": 1048510, "threadCount": 1 }, - "alsologtostderr": true + "alsologtostderr": true, + "stdLogLevel": "DEBUG", + "syncFlush": true } )"; diff --git a/functionsystem/tests/unit/main.cpp b/functionsystem/tests/unit/main.cpp index dbb33f6..49de462 100644 --- a/functionsystem/tests/unit/main.cpp +++ b/functionsystem/tests/unit/main.cpp @@ -42,7 +42,8 @@ const std::string LOG_CONFIG_JSON = R"( "threadCount": 1 }, "alsologtostderr": true, - "stdLogLevel": "DEBUG" + "stdLogLevel": "DEBUG", + "syncFlush": true } )"; diff --git a/scripts/executor/run_code_gate.py b/scripts/executor/run_code_gate.py new file mode 100644 index 0000000..fa17a67 --- /dev/null +++ b/scripts/executor/run_code_gate.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd +import os +import re +import utils +import json +import time +import socket +import argparse +import subprocess +from urllib.request import urlopen +from urllib.request import Request +from datetime import datetime, timezone + +from concurrent.futures import ProcessPoolExecutor + +log = utils.init_logger() +_env = utils.pipeline_env() +_exec_timeout = int(os.getenv("EXEC_TIMEOUT", 120)) # 单个测试用例执行超时时间，单位秒 + +def decode_args(): + """ + 解析脚本输入参数 + :return: IT用例程序，UT用例程序，指定的测试套，指定的测试用例 + """ + parser = argparse.ArgumentParser(description="Test unit executor") + parser.add_argument("--it_bin", type=str, help="Integrate test executable path", required=True) + parser.add_argument("--ut_bin", type=str, help="Unit test executable path", required=True) + parser.add_argument("--test_suite", type=str, help="Add test_suite in gtest_filter", required=True) + parser.add_argument("--test_case", type=str, help="Add test_case in gtest_filter", required=True) + args = parser.parse_args() + + return args.it_bin, args.ut_bin, args.test_suite, args.test_case + + +def test_controller(it_bin, ut_bin, test_suite, test_case): + log.info(f"Read IT test executable path: {it_bin}") + log.info(f"Read UT test executable path: {ut_bin}") + log.info(f"Read test suit in gtest_filter: {test_suite}") + log.info(f"Read test case in gtest_filter: {test_case}") + start_time = time.time() + + # 采集所有测试用例的名字 + collector = { + "test_suite": test_suite, + "test_case": test_case, + "it_bin": it_bin, + "it_count": 0, + "it_index": gather_tests_index(it_bin), + "ut_bin": ut_bin, + "ut_count": 0, + "ut_index": gather_tests_index(ut_bin), + "merge_info": { + "user": os.getenv("codehubMergedByUser"), + "repo": os.getenv("codehubSourceRepoName"), + "merge": os.getenv("codehubMergeRequestIid"), + "branch": os.getenv("codehubTargetBranch"), + "target": os.getenv("codehubTargetRepoSshUrl"), + } + } + + for key, value in collector["it_index"].items(): + log.info(f"Find IT test-suit[{key}] cases: {value}") + collector["it_count"] += len(value) + + for key, value in collector["ut_index"].items(): + log.info(f"Find UT test-suit[{key}] cases: {value}") + collector["ut_count"] += len(value) + + # 执行单个/单组测试用例 + if test_suite != "*": + if test_case == "": + test_case = "*" + if test_suite in collector["it_index"].keys(): + proc_result = run_test_unit(it_bin, test_suite, test_case) + case_results = parse_test_output(*proc_result) + return summarize(start_time, collector, [proc_result], case_results) + elif test_suite in collector["ut_index"].keys(): + proc_result = run_test_unit(ut_bin, test_suite, test_case) + case_results = parse_test_output(*proc_result) + return summarize(start_time, collector, [proc_result], case_results) + else: + log.error(f"Target test-unit: {test_suite}.{test_case} not found.") + return 0 + + # 执行全部测试用例 + cpu_count, total_mem = utils.get_linux_resources() + worker_num = min(int(os.getenv("MAX_WORKERS", 64)), cpu_count) + log.info(f"Run all test suites in {worker_num} workers. Total resources available: {cpu_count}CPU {total_mem}GB") + executor = ProcessPoolExecutor(max_workers=worker_num) + futures = [] + for test_suite in sorted(collector["it_index"].keys()): + futures.append(executor.submit(run_test_unit_proc, it_bin, test_suite, "*")) + for test_suite in sorted(collector["ut_index"].keys()): + futures.append(executor.submit(run_test_unit_proc, ut_bin, test_suite, "*")) + + proc_results = [] + case_results = [] + for future in futures: + result = future.result() + case_results.extend(parse_test_output(*result)) + proc_results.append(result) + + return summarize(start_time, collector, proc_results, case_results) + + +def gather_tests_index(path): + # 先收集非期望的输出 + _, main_output, _ = run_command(path, "--gtest_color=no", "--gtest_list_tests", "--gtest_filter=-*") + main_output = main_output.split("\n") + + _, tests_list, _ = run_command(path, "--gtest_color=no", "--gtest_list_tests") + tests_list = tests_list.split("\n") + + it = 0 # 虚拟指针指向非期望的输出内容的行 + tests_index = {} + test_suite_name = "" + log_data = datetime.now(timezone.utc).strftime("%m%d") + for line in tests_list: + if (line.startswith(f"D{log_data}") or line.startswith(f"I{log_data}") + or line.startswith(f"W{log_data}") or line.startswith(f"E{log_data}")): + it += 1 + continue # 跳过元戎日志输出 + elif main_output[it] == line: + it += 1 + continue # 跳过内容一直的输出 + elif line.startswith(" "): + # 记录测试用例名 + test_case = line[2:].split(" ")[0] + tests_index.setdefault(test_suite_name, []).append(test_case) + else: + # 记录测试套名称 + test_suite_name = line[:-1] + + return tests_index + + +def run_command(path, *args): + cmd = [path, *args] + log.info(f"Run command: {' '.join(cmd)}") + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, cmd, output=proc.stdout, stderr=proc.stderr) + + return proc.returncode, proc.stdout, proc.stderr + + +def run_test_unit_proc(path, test_suite, test_case): + """ + 用于在并发场景下包装测试用例执行函数，捕获异常进程信息 + :return: 进程名，执行时间，退出代码，[标准输入，标准输出，标准错误] + """ + proc_name = path.split('/')[-1] + proc_desc = f"<{proc_name}>[{test_suite}.{test_case}]" + + retry_times = max(1, int(os.getenv("RETRY_TIMES", "1"))) + try: + while retry_times > 0: + result = run_test_unit(path, test_suite, test_case) + retry_times -= 1 + if result[2] == 0: + return result + elif retry_times <= 0: + return result + log.warning(f"Retry to run test {proc_desc}. {retry_times} retry times left.") + except Exception as e: # 捕获所有异常，避免子进程异常退出导致主进程异常终止 + log.error(f"Run test {proc_desc} error: {e}") + return proc_name, 0, -400, [[test_suite, test_case], "", str(e)] + + +def run_test_unit(path, test_suite, test_case): + """ + 测试用例执行函数，捕获程序执行状态、标准输出、标准错误 + :return: 进程名，执行时间，退出代码，[标准输入，标准输出，标准错误] + """ + proc_name = path.split('/')[-1] + proc_desc = f"<{proc_name}>[{test_suite}.{test_case}]" + log.info(f"Start test process: {proc_desc}") + start_time = time.time() + try: + proc = subprocess.run( + [path, "--gtest_color=no", "--gtest_print_time=1", f"--gtest_filter={test_suite}.{test_case}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + timeout=_exec_timeout, + errors="replace", + text=True, + ) + except subprocess.TimeoutExpired as e: # 捕获子进程超时异常 + log.error(f"Run test {proc_desc} timeout: {e}") + stdout = "" if e.stdout is None else e.stdout.decode("utf-8", errors="replace") + stderr = "" if e.stderr is None else e.stderr.decode("utf-8", errors="replace") + return proc_name, _exec_timeout * 1000, -408, [[test_suite, test_case], stdout, stderr] + stop_time = time.time() + exec_time = int((stop_time - start_time) * 1000) # 转换为毫秒 + log.info(f"Test process {proc_desc} finished in {exec_time}ms with code {proc.returncode}.") + return proc_name, exec_time, proc.returncode, [[test_suite, test_case], proc.stdout, proc.stderr] + + + + +def parse_test_output(proc_name, exec_time, exit_code, std): + """ + 串行打印并分析测试用例的输出信息 + """ + proc_desc = f"<{proc_name}>[{std[0][0]}.{std[0][1]}]" + stdout_lines = std[1].split("\n") + stderr_lines = std[2].split("\n") + + test_log_print_disable = os.getenv("TEST_LOG_PRINT_DISABLE") + if test_log_print_disable != "true": + log.info(f"Show process {proc_desc} original standard output and standard errors") + log.info(f"{'-' * 10} {proc_desc} stdout {'-' * 10}") + for line in stdout_lines: + log.info(line) + log.info(f"{'-' * 10} {proc_desc} stderr {'-' * 10}") + for line in stderr_lines: + log.error(line) + log.info(f"{'-' * 10} {proc_desc} EOF {'-' * 10}") + + results = [] + running = False + for line in stdout_lines: + if line.startswith("[==========]"): + running = not running + elif running and line.startswith("[ OK ]"): + result = test_result_line("OK", line, std[0][0], std[0][1]) + if result is None: + continue + result.append(proc_name) + results.append(result) + elif running and line.startswith("[ FAILED ]"): + result = test_result_line("FAILED", line, std[0][0], std[0][1]) + if result is None: + continue + result.append(proc_name) + results.append(result) + + log.info(f"Parallel controller receive process {proc_desc} exit code {exit_code} in {exec_time}ms. " + + f"Receive stdout {len(stdout_lines)} lines and stderr {len(stderr_lines)} lines") + + return results + + +pattern = r'\[\s+(\w+)\s+\]\s+([\w\/]+)\.([\w\/]+)\s+\((\d+)\s+ms\)' + + +def test_result_line(hits: str , line: str, test_suite="Unknown", test_case="Unknown"): + matches = re.match(pattern, line) + if not matches: + log.error(f"Invalid test result line: {line}") + return [hits, test_suite, test_case, "-1"] + return [matches.group(1), matches.group(2), matches.group(3), + int(matches.group(4))] # status, test_suite, test_case, exec_time + + +def summarize(start_time, collector, proc_results, case_results): + log.info("Summarize the test component run results") + + log.info(f"The IT has a total of {len(collector['it_index'])} test suites and {collector['it_count']} test cases") + log.info(f"The UT has a total of {len(collector['ut_index'])} test suites and {collector['ut_count']} test cases") + + if os.getenv("LOGSTASH_URL") is None: + log.warning("LOGSTASH_URL is not set. Can not send any message to logstash.") + + core_dump = False + # 统计进程运行状态 + for result in proc_results: + proc_name, exec_time, exit_code, std = result + ex = { + "proc_name": proc_name, + "exec_time": exec_time, + "test_suite": std[0][0], + "test_case": std[0][1], + "status": "OK" if exit_code == 0 else "FAILED", + "exit_code": exit_code, + "stdout_len": len(std[1]), + "stderr_len": len(std[2]), + } + proc_desc = f"<{proc_name}>[{std[0][0]}.{std[0][1]}]" + if exit_code != 0: + info = f"Run test suite {proc_desc} failed[{exit_code}] in {exec_time}ms." + core_dump = True + log.error(info) + else: + info = f"Run test suite {proc_desc} success[{exit_code}] in {exec_time}ms." + log2es("TestSuite", info, collector["merge_info"], ex) + + # 统计用例运行状态 + for result in case_results: + if result is None: + continue + status, test_suite, test_case, exec_time, proc_name = result + ex = { + "proc_name": proc_name, + "exec_time": exec_time, + "test_suite": test_suite, + "test_case": test_case, + "status": status, + } + case_desc = f"<{proc_name}>[{test_suite}.{test_case}]" + if status != "OK": + info = f"Run test case {case_desc} failed[{status}] in {exec_time}ms." + core_dump = True + log.error(info) + else: + info = f"Run test case {case_desc} success[{status}] in {exec_time}ms." + log2es("TestCase", info, collector["merge_info"], ex) + + stop_time = time.time() + dt_time = (stop_time - start_time) * 1000 + ex = { + "exec_time": int(dt_time), + "test_suite": "*", + "test_case": "*", + } + if core_dump: + ex["status"] = "FAILED" + info = "An error occurred during test case execution" + log2es("Summarize", info, collector["merge_info"], ex) + log.error(info) + return 1 + + ex["status"] = "OK" + info = "All test suit executed successfully" + log2es("Summarize", info, collector["merge_info"], ex) + log.info(info) + return 0 + + +def log2es(step: str, text: str, info: dict, ex: dict = None): + data = { + "step": step, + "info": info, + "text": text, + "envs": _env + } + data.update(ex) + + headers = { + 'X-SubSystem': "function_system", + 'X-Pipeline-Name': _env["JOB_BASE_NAME"], + 'X-Pipeline-Type': "gate", + 'X-Pipeline-Step': "gtest", + } + result = utils.report2es(headers, data) + if result is not None: + log.error(result) + + +if __name__ == '__main__': + _args = decode_args() + _start_time = time.time() + _code = test_controller(*_args) + _stop_time = time.time() + _exec_time = _stop_time - _start_time + log.info(f"All test-suit executed finished in {_exec_time * 1000}ms") + exit(_code) diff --git a/scripts/executor/utils/__init__.py b/scripts/executor/utils/__init__.py new file mode 100644 index 0000000..d305d43 --- /dev/null +++ b/scripts/executor/utils/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd +from .tools import init_logger +from .tools import pipeline_env +from .tools import get_linux_resources +from .tools import report2es diff --git a/scripts/executor/utils/tools.py b/scripts/executor/utils/tools.py new file mode 100644 index 0000000..333e017 --- /dev/null +++ b/scripts/executor/utils/tools.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd +import os +import sys +import json +import socket +import logging +from typing import Tuple +from typing import Optional +from urllib.request import urlopen +from urllib.request import Request + + +# 请勿在此声明全局变量 + +def init_logger() -> logging.Logger: + """ + 初始化日志组件 + """ + logging.basicConfig( + level=logging.INFO, + format='[%(levelname)s][%(asctime)s] %(message)s', + datefmt='%b %d %H:%M:%S', + handlers=[logging.StreamHandler(sys.stdout)] + ) + log = logging.getLogger() + return log + + +def pipeline_env() -> dict: + """ + 采集流水线环境变量信息 + """ + return { + "BUILD_ID": os.getenv("BUILD_ID", ""), + "BUILD_URL": os.getenv("BUILD_URL", ""), + "JOB_URL": os.getenv("JOB_URL", ""), + "JOB_BASE_NAME": os.getenv("JOB_BASE_NAME", ""), + "RUN_DISPLAY_URL": os.getenv("RUN_DISPLAY_URL", ""), + } + + +def get_linux_resources() -> Tuple[int, int]: + """ + 采集Linux环境CPU和MEM信息 + """ + with open("/proc/cpuinfo") as f: + cpu_count = sum(1 for line in f if line.startswith("processor")) + + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemTotal"): + total_mem_kb = int(line.split()[1]) + break + + total_mem_gb = round(total_mem_kb / (1024 ** 2), 2) + return cpu_count, total_mem_gb + + +def report2es(headers: dict, data: dict) -> Optional[str]: + """ + 上报事件到ES存储 + """ + url = os.getenv("LOGSTASH_URL") + if url is None: + return None + + headers.update({ + 'Content-Type': 'application/json', + 'User-Agent': socket.gethostname(), + }) + json_data = json.dumps(data).encode("utf-8") + req = Request(url=url, method="POST", headers=headers, data=json_data) + + try: + rsp = urlopen(req) + except (ConnectionRefusedError, ConnectionResetError) as e: + return f"Connect [{url}] error: {e}" + status_code = rsp.status + if status_code != 200: + return f"Failed to send logstash data: {data}" + return None -- Gitee From acd2a7b7508bc366eb3a2308ea36b664d804e8b8 Mon Sep 17 00:00:00 2001 From: wolfbolin Date: Thu, 20 Nov 2025 10:46:22 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E3=80=90CICD=E3=80=91=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=E8=84=9A=E6=9C=AC=E6=89=A7=E8=A1=8C=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E4=B8=BAMakefile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.sh => Makefile | 32 +- build.sh | 170 ---------- functionsystem/build.sh | 3 +- scripts/executor/.gitignore | 2 + scripts/executor/make_functionsystem.py | 53 +++ scripts/executor/tasks/__init__.py | 10 + scripts/executor/tasks/build_task.py | 82 +++++ scripts/executor/tasks/clean_task.py | 16 + .../download_thirdparty.py} | 4 +- scripts/executor/tasks/test_task.py | 13 + scripts/executor/utils/__init__.py | 3 + scripts/executor/utils/process.py | 39 +++ thirdparty/yuanrong/cmake/datasystem.cmake | 36 -- thirdparty/yuanrong/cmake/litebus.cmake | 35 -- thirdparty/yuanrong/cmake/logs.cmake | 40 --- .../cmake/observability_metrics.cmake | 43 --- tools/utils.sh | 317 ------------------ 17 files changed, 232 insertions(+), 666 deletions(-) rename config.sh => Makefile (48%) delete mode 100755 build.sh create mode 100644 scripts/executor/.gitignore create mode 100644 scripts/executor/make_functionsystem.py create mode 100644 scripts/executor/tasks/__init__.py create mode 100644 scripts/executor/tasks/build_task.py create mode 100644 scripts/executor/tasks/clean_task.py rename scripts/executor/{download_opensource.py => tasks/download_thirdparty.py} (95%) create mode 100644 scripts/executor/tasks/test_task.py create mode 100644 scripts/executor/utils/process.py delete mode 100644 thirdparty/yuanrong/cmake/datasystem.cmake delete mode 100644 thirdparty/yuanrong/cmake/litebus.cmake delete mode 100644 thirdparty/yuanrong/cmake/logs.cmake delete mode 100644 thirdparty/yuanrong/cmake/observability_metrics.cmake diff --git a/config.sh b/Makefile similarity index 48% rename from config.sh rename to Makefile index 1e38db3..ddce69f 100755 --- a/config.sh +++ b/Makefile @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,28 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set -eo pipefail -GIT_BRANCH=PD_develop -BASE_DIR=$( - cd "$(dirname "$0")" - pwd -) -CPU_NUM="$(grep -c 'processor' /proc/cpuinfo)" -JOB_NUM="$(($(grep -c 'processor' /proc/cpuinfo) + 1))" -YR_FUNCTION_SYSTEM_VERSION="0.0.1" -OUTPUT_DIR=$(realpath "$BASE_DIR"/../output) -echo "OUTPUT_DIR: ${OUTPUT_DIR}" +.PHONY: help build clean -USAGE=" -Usage: bash build.sh [-h] [-v] [-c] [-j ]\n\n +help: + python3 ./scripts/executor/make_functionsystem.py -h -Options:\n - -h show usage\n - -v version\n - -c clean vendor downloaded cache\n\n - -j set the number of jobs run in parallel for compiling, for example: the number of processor(${CPU_NUM}). See build docs for details.\n\n +build: + python3 ./scripts/executor/make_functionsystem.py build -Example:\n - bash build.sh -v 0.0.1 -j 16 -" +clean: + python3 ./scripts/executor/make_functionsystem.py clean + +test: + python3 ./scripts/executor/make_functionsystem.py test diff --git a/build.sh b/build.sh deleted file mode 100755 index c11e7c0..0000000 --- a/build.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -source /etc/profile.d/*.sh -BASE_DIR=$(dirname "$(readlink -f "$0")") -CPU_NUM="$(grep -c 'processor' /proc/cpuinfo)" - -. "${BASE_DIR}"/config.sh -. "${BASE_DIR}"/tools/utils.sh - -function thirdparty_compile() { - python ${BASE_DIR}/scripts/executor/download_opensource.py --config ${BASE_DIR}/vendor/ThirdPartyList.csv --output ${BASE_DIR}/vendor/src - cd "${BASE_DIR}"/vendor - cmake -B build - cmake --build build --parallel ${JOB_NUM} - - # compile thirdparty: etcd - cd "${BASE_DIR}"/scripts/ - bash basic_build.sh || die "etcd build failed" -} - -function logs_compile() { - echo "Compile common/logs" - cd "${BASE_DIR}"/common/logs - bash build.sh -j "${JOB_NUM}" -} - -function litebus_compile() { - echo "Compile common/litebus" - cd "${BASE_DIR}"/common/litebus - bash build.sh -t off -j "${JOB_NUM}" -} - -function metrics_compile() { - echo "Compile common/metrics" - cd "${BASE_DIR}"/common/metrics - bash build.sh -j "${JOB_NUM}" -} - -function functionsystem_compile() { - cd "${BASE_DIR}"/functionsystem - # compile functionsystem - bash build.sh -r -j "${JOB_NUM}" -v "yr-functionsystem-v${YR_FUNCTION_SYSTEM_VERSION}" - - # compile functioncore - bash build.sh -y -j "${JOB_NUM}" -v "yr-functionsystem-v${YR_FUNCTION_SYSTEM_VERSION}" - - unset CMC_USERNAME - unset CMC_PASSWORD -} - -function package() { - bash "${BASE_DIR}"/scripts/package/package.sh $@ -} - -function clean_build_cache() { - echo "clean vendor cache and component build cache" - cd "${BASE_DIR}" && git clean -dffx - git submodule foreach --recursive git clean -dffx -} - -function clean() { - [[ -n "${OUTPUT_DIR}" ]] && rm -rf "${OUTPUT_DIR}" -} - -function doc_build() { - pushd ${BASE_DIR} - pip install -r runtime/api/python/requirements.txt - pip install -r docs/requirements_dev.txt - popd - - pushd ${BASE_DIR}/docs - make html - # disable configuration：SPHINXOPTS="-W --keep-going -n", enable it after all alarms are cleared. - popd - - # modify sphinx(7.3.7) build-in search，open limit on numbers in search. - # Changes in later versions need to be modified accordingly. - sed -i '285d' "${BASE_DIR}"/docs/_build/html/_static/searchtools.js - sed -i '284s/ ||//' "${BASE_DIR}"/docs/_build/html/_static/searchtools.js - rm -rf "${OUTPUT_DIR}"/docs && mkdir -p "${OUTPUT_DIR}"/docs - cp -rf "${BASE_DIR}"/docs/_build/html/* "${OUTPUT_DIR}"/docs -} - -function check_number() { - number_check='^([0-9]+)$' - if [[ "$1" =~ ${number_check} ]]; then - return 0 - else - log_error "Invalid value $1 for option -$2" - log_warning "${USAGE}" - exit 1 - fi -} - -function build() { - while getopts 'j:v:hc' opt; do - case "$opt" in - j) - check_number "${OPTARG}" j - if [ ${OPTARG} -gt $(($CPU_NUM * 2)) ]; then - log_warning "The -j $OPTARG is over the max logical cpu count($CPU_NUM) * 2" - fi - JOB_NUM="${OPTARG}" - ;; - v) - YR_FUNCTION_SYSTEM_VERSION=${OPTARG} - ;; - h) - echo -e $USAGE - exit 0 - ;; - c) - clean_build_cache - exit 0 - ;; - *) - log_error "Invalid command" - exit 1 - ;; - esac - done - - log_info "compile functionsystem" - thirdparty_compile - logs_compile - litebus_compile - metrics_compile - functionsystem_compile -} - -# provide subcommands so we can support more scenario -case "$1" in - clean) - shift - clean $@ - clean_build_cache - ;; - build) - shift - build $@ - ;; - package) - shift - package $@ - ;; - doc-build) - shift - doc_build $@ - ;; - *) - log_info "Using default subcommand: build" - clean - build $@ - ;; -esac diff --git a/functionsystem/build.sh b/functionsystem/build.sh index aff83ae..f9b4509 100644 --- a/functionsystem/build.sh +++ b/functionsystem/build.sh @@ -261,7 +261,8 @@ while getopts 'yghrxVbm:v:o:j:S:Cc:u:t:M:d:T:s:R:P:p:k' opt; do fi ;; v) - YR_VERSION="${OPTARG}" + VERSION="${OPTARG}" + YR_VERSION="yr-functionsystem-v${VERSION}" ;; V) VERBOSE="-v" diff --git a/scripts/executor/.gitignore b/scripts/executor/.gitignore new file mode 100644 index 0000000..f4882dc --- /dev/null +++ b/scripts/executor/.gitignore @@ -0,0 +1,2 @@ +*/__pycache__/ +*.pyc \ No newline at end of file diff --git a/scripts/executor/make_functionsystem.py b/scripts/executor/make_functionsystem.py new file mode 100644 index 0000000..352cdaf --- /dev/null +++ b/scripts/executor/make_functionsystem.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import sys +import tasks +import utils +import argparse +import subprocess + +log = utils.init_logger() +# 计算根目录，期望的目录结构为 yuanrong-functionsystem/scripts/executor/make_functionsystem.py +ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..")) + + +def parser_args(): + parser = argparse.ArgumentParser(description="Build script for function system") + subparsers = parser.add_subparsers(dest="subcommand", required=False) + + build_parser = subparsers.add_parser("build", help="Build all components of function system") + build_parser.add_argument("-j", "--jobs", type=int, help="Set the number of jobs run in parallel for compiling") + build_parser.add_argument("-v", "--version", type=str, help="Set the version for function system build") + build_parser.set_defaults(func=lambda args: tasks.run_build(ROOT_DIR, args)) + + clean_parser = subparsers.add_parser("clean", help="Clean all build artifacts and caches") + clean_parser.set_defaults(func=lambda args: tasks.run_clean(ROOT_DIR, args)) + + test_parsor = subparsers.add_parser("test", help="Run tests for function system") + test_parsor.set_defaults(func=lambda args: tasks.run_test(ROOT_DIR, args)) + + args = parser.parse_args() + return parser, args + +if __name__ == "__main__": + _parser, _args = parser_args() + # 执行对应的函数 + if hasattr(_args, 'func'): + _args.func(_args) + else: + _parser.print_help() diff --git a/scripts/executor/tasks/__init__.py b/scripts/executor/tasks/__init__.py new file mode 100644 index 0000000..d53e0ef --- /dev/null +++ b/scripts/executor/tasks/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd +from .build_task import run_build +from .download_thirdparty import download_thirdparty + +from .clean_task import run_clean + +from .test_task import run_test + diff --git a/scripts/executor/tasks/build_task.py b/scripts/executor/tasks/build_task.py new file mode 100644 index 0000000..126e157 --- /dev/null +++ b/scripts/executor/tasks/build_task.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd + +import os +import utils +import tasks + +log = utils.init_logger() + + +def run_build(ROOT_DIR, args): + build_args = { + "root_dir": ROOT_DIR, + "job_num": args.jobs if args.jobs else (os.cpu_count() or 1), + "version": args.version if args.version else "0.0.1" + } + if build_args['job_num'] > (os.cpu_count() or 1) * 2: + log.warning(f"The -j {build_args['job_num']} is over the max logical cpu count({os.cpu_count()}) * 2") + log.info(f"Start to build function system with args: {build_args}") + + compile_thirdparty(build_args) + compile_logs(build_args) + compile_litebus(build_args) + compile_metrics(build_args) + compile_functionsystem(build_args) + +def compile_thirdparty(args): + print(args['root_dir'],args['job_num']) + + # 根据下载清单下载第三方依赖 + tasks.download_thirdparty( + config_path = os.path.join(args['root_dir'], "vendor/ThirdPartyList.csv"), + download_path = os.path.join(args['root_dir'], "vendor/src") + ) + + utils.sync_command( + ["cmake", "-B", "build"], + cwd=os.path.join(args['root_dir'], "vendor") + ) + utils.sync_command( + ["cmake", "--build", "build", "--parallel", str(args['job_num'])], + cwd=os.path.join(args['root_dir'], "vendor") + ) + utils.sync_command( + ["bash", "basic_build.sh"], + cwd=os.path.join(args['root_dir'], "scripts") + ) + +def compile_logs(args): + log.info("Start to compile common/logs") + utils.sync_command( + ["bash", "build.sh", "-j", str(args['job_num'])], + cwd=os.path.join(args['root_dir'], "common", "logs") + ) + +def compile_litebus(args): + log.info("Start to compile common/litebus") + utils.sync_command( + ["bash", "build.sh", "-t", "off", "-j", str(args['job_num'])], + cwd=os.path.join(args['root_dir'], "common", "litebus") + ) + +def compile_metrics(args): + log.info("Start to compile common/metrics") + utils.sync_command( + ["bash", "build.sh", "-j", str(args['job_num'])], + cwd=os.path.join(args['root_dir'], "common", "metrics") + ) + +def compile_functionsystem(args): + cwd = os.path.join(args['root_dir'], "functionsystem") + log.info("Start to compile functionsystem") + utils.sync_command( + ["bash", "build.sh", "-r", "-j", str(args['job_num']), "-v", args['version']], + cwd=cwd + ) + utils.sync_command( + ["bash", "build.sh", "-y", "-j", str(args['job_num']), "-v", args['version']], + cwd=cwd + ) + diff --git a/scripts/executor/tasks/clean_task.py b/scripts/executor/tasks/clean_task.py new file mode 100644 index 0000000..60affcb --- /dev/null +++ b/scripts/executor/tasks/clean_task.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd + +import os +import utils +import tasks + +log = utils.init_logger() + + +def run_clean(ROOT_DIR, args): + log.info("Start to clean vendor cache and component build cache") + utils.sync_command(["git", "clean", "-dffx"], cwd=ROOT_DIR) + utils.sync_command(["git", "submodule", "foreach", "--recursive", "git clean -dffx"], cwd=ROOT_DIR) + diff --git a/scripts/executor/download_opensource.py b/scripts/executor/tasks/download_thirdparty.py similarity index 95% rename from scripts/executor/download_opensource.py rename to scripts/executor/tasks/download_thirdparty.py index da8b077..ff78f4a 100644 --- a/scripts/executor/download_opensource.py +++ b/scripts/executor/tasks/download_thirdparty.py @@ -9,7 +9,7 @@ import subprocess import urllib.request -def download_opensource(config_path, download_path): +def download_thirdparty(config_path, download_path): """主函数""" config_path = os.path.abspath(config_path) if not os.path.exists(config_path): @@ -131,5 +131,5 @@ if __name__ == "__main__": _parser.add_argument('--config', type=str, required=True, help='配置文件路径') _parser.add_argument('--output', type=str, required=True, help='依赖下载路径') _args = _parser.parse_args() - _code = download_opensource(_args.config, _args.output) + _code = download_thirdparty(_args.config, _args.output) exit(_code) diff --git a/scripts/executor/tasks/test_task.py b/scripts/executor/tasks/test_task.py new file mode 100644 index 0000000..99a8e9c --- /dev/null +++ b/scripts/executor/tasks/test_task.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd + +import os +import utils +import tasks + +log = utils.init_logger() + + +def run_test(ROOT_DIR, args): + log.error("Test task is not implemented yet.") \ No newline at end of file diff --git a/scripts/executor/utils/__init__.py b/scripts/executor/utils/__init__.py index d305d43..c4ff014 100644 --- a/scripts/executor/utils/__init__.py +++ b/scripts/executor/utils/__init__.py @@ -5,3 +5,6 @@ from .tools import init_logger from .tools import pipeline_env from .tools import get_linux_resources from .tools import report2es + +from .process import sync_command +from .process import exec_command \ No newline at end of file diff --git a/scripts/executor/utils/process.py b/scripts/executor/utils/process.py new file mode 100644 index 0000000..bb282fa --- /dev/null +++ b/scripts/executor/utils/process.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# coding=UTF-8 +# Copyright (c) 2025 Huawei Technologies Co., Ltd + +import os +import subprocess + +def sync_command(cmd: tuple[str], cwd:str = None) -> tuple: + """ + 执行指定路径的命令，并返回命令的返回码、标准输出和标准错误。 + 注意：不能打印cmd，因为cmd可能包含敏感信息或用户输入的参数，打印可能导致信息泄露。 + + :param process: 可执行命令路径 + :type process: str + :param args: 命令的参数 + :type args: tuple + :return: 返回码、标准输出和标准错误 + :rtype: tuple + """ + print(f"Executing command[{cmd[0]}] in {cwd if cwd else os.getcwd()}") + proc = subprocess.run( + cmd, + cwd=cwd, + text=True, + ) + + +def exec_command(cmd: tuple[str]): + """ + 启动指定的shell命令，并用该命令替换当前进程，类似于shell中的exec命令。 + + :param process: 可执行命令路径 + :type process: str + :param args: 命令的参数 + :type args: tuple + """ + # 使用os.execvp来替换当前进程 + os.execvp(cmd[0], cmd[1:]) + diff --git a/thirdparty/yuanrong/cmake/datasystem.cmake b/thirdparty/yuanrong/cmake/datasystem.cmake deleted file mode 100644 index 4e9c5b6..0000000 --- a/thirdparty/yuanrong/cmake/datasystem.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(src_dir ${ROOT_DIR}/datasystem) - -get_filename_component(absolute_src_dir ${src_dir} ABSOLUTE) -set(src_dir ${absolute_src_dir}) - -set(datasystem_INCLUDE_DIR ${src_dir}/output/sdk/cpp/include) -set(datasystem_LIB_DIR ${src_dir}/output/sdk/cpp/lib) -set(datasystem_LIB ${datasystem_LIB_DIR}/libdatasystem.so) - -include_directories(${datasystem_INCLUDE_DIR}) - -message(STATUS "datasystem lib dir: ${datasystem_LIB_DIR}") -file(GLOB DS_SPDLOG "${datasystem_LIB_DIR}/libds-spdlog.so*") -file(GLOB TBB "${datasystem_LIB_DIR}/libtbb*") -file(GLOB ZMQ "${datasystem_LIB_DIR}/libzmq*") -file(GLOB DS_CURL "${datasystem_LIB_DIR}/libcurl*") -install(FILES ${datasystem_LIB_DIR}/libdatasystem.so DESTINATION ${INSTALL_LIBDIR}) -install(FILES ${DS_SPDLOG} DESTINATION ${INSTALL_LIBDIR}) -install(FILES ${TBB} DESTINATION ${INSTALL_LIBDIR}) -install(FILES ${ZMQ} DESTINATION ${INSTALL_LIBDIR}) -install(FILES ${DS_CURL} DESTINATION ${INSTALL_LIBDIR}) -install(FILES ${datasystem_LIB_DIR}/libacl_plugin.so DESTINATION ${INSTALL_LIBDIR} OPTIONAL) \ No newline at end of file diff --git a/thirdparty/yuanrong/cmake/litebus.cmake b/thirdparty/yuanrong/cmake/litebus.cmake deleted file mode 100644 index 3757202..0000000 --- a/thirdparty/yuanrong/cmake/litebus.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(src_dir ${ROOT_DIR}/common/litebus) -get_filename_component(absolute_src_dir ${src_dir} ABSOLUTE) -set(src_dir ${absolute_src_dir}) - -message(STATUS "litebus src dir: ${src_dir}") -if (NOT EXISTS ${src_dir}/output) - if (EXISTS ${src_dir}) - message(STATUS "begin build litebus on ${src_dir}") - execute_process(COMMAND bash build/build.sh -X 1.1.1 -W off -t off -x ${DOWNLOAD_OPENSRC} -T ${THIRDPARTY_SRC_DIR} WORKING_DIRECTORY ${src_dir}) - endif() -endif() - - -set(litebus_INCLUDE_DIR ${src_dir}/output/include) -set(litebus_LIB_DIR ${src_dir}/output/lib) -set(litebus_LIB ${litebus_LIB_DIR}/liblitebus.so) - -include_directories(${litebus_INCLUDE_DIR}) - -install(FILES ${litebus_LIB_DIR}/liblitebus.so DESTINATION lib) -install(FILES ${litebus_LIB_DIR}/liblitebus.so.0.0.1 DESTINATION lib) \ No newline at end of file diff --git a/thirdparty/yuanrong/cmake/logs.cmake b/thirdparty/yuanrong/cmake/logs.cmake deleted file mode 100644 index c737a94..0000000 --- a/thirdparty/yuanrong/cmake/logs.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(src_name yrlogs) -set(src_dir ${ROOT_DIR}/common/logs) -get_filename_component(ABSOLUTE_src_dir ${src_dir} ABSOLUTE) -set(src_dir ${ABSOLUTE_src_dir}) - -message(STATUS "logs src dir: ${src_dir}") - -if (NOT EXISTS ${src_dir}/output) - if (EXISTS ${src_dir}) - message(STATUS "begin build yrlogs, download opensrc: ${DOWNLOAD_OPENSRC}, thirdparty src dir: ${THIRDPARTY_SRC_DIR}") - execute_process(COMMAND bash build.sh -x ${DOWNLOAD_OPENSRC} -T ${THIRDPARTY_SRC_DIR} WORKING_DIRECTORY ${src_dir}) - endif() -endif() - -set(${src_name}_INCLUDE_DIR ${src_dir}/output/include) -set(${src_name}_LIB_DIR ${src_dir}/output/lib) -set(${src_name}_LIB ${${src_name}_LIB_DIR}/libyrlogs.so ${${src_name}_LIB_DIR}/libspdlog.so) - -include_directories(${${src_name}_INCLUDE_DIR}) - -message(STATUS "yrlogs include dir: ${${src_name}_INCLUDE_DIR}") - -install(FILES ${${src_name}_LIB_DIR}/libyrlogs.so DESTINATION lib) -install(FILES ${${src_name}_LIB_DIR}/libspdlog.so DESTINATION lib) -install(FILES ${${src_name}_LIB_DIR}/libspdlog.so.1.12 DESTINATION lib) -install(FILES ${${src_name}_LIB_DIR}/libspdlog.so.1.12.0 DESTINATION lib) \ No newline at end of file diff --git a/thirdparty/yuanrong/cmake/observability_metrics.cmake b/thirdparty/yuanrong/cmake/observability_metrics.cmake deleted file mode 100644 index 2912f95..0000000 --- a/thirdparty/yuanrong/cmake/observability_metrics.cmake +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(src_dir ${ROOT_DIR}/common/metrics) -get_filename_component(ABSOLUTE_src_dir ${src_dir} ABSOLUTE) -set(src_dir ${ABSOLUTE_src_dir}) - -message(STATUS "metrics src dir: ${src_dir}") -if (NOT EXISTS ${src_dir}/output) - if (EXISTS ${src_dir}) - message(STATUS "begin build metrics") - execute_process(COMMAND bash build.sh -x ${DOWNLOAD_OPENSRC} -T ${THIRDPARTY_SRC_DIR} WORKING_DIRECTORY ${src_dir}) - endif() -endif() - -set(metrics_INCLUDE_DIR ${src_dir}/output/include) -set(metrics_LIB_DIR ${src_dir}/output/lib) -set(metrics_LIB ${metrics_LIB_DIR}/libobservability-metrics.so) -set(metrics_sdk_LIB ${metrics_LIB_DIR}/libobservability-metrics-sdk.so) -set(metrics_file_exporter_LIB ${metrics_LIB_DIR}/libobservability-metrics-file-exporter.so) -set(metrics_exporter_ostream_LIB ${metrics_LIB_DIR}/libobservability-metrics-exporter-ostream.so) - -include_directories(${metrics_INCLUDE_DIR}) - -message(STATUS "metrics include dir: ${metrics_INCLUDE_DIR}") - -install(FILES - ${metrics_LIB} - ${metrics_sdk_LIB} - ${metrics_file_exporter_LIB} - ${metrics_exporter_ostream_LIB} - DESTINATION lib) diff --git a/tools/utils.sh b/tools/utils.sh index db9a1eb..eafa5ed 100644 --- a/tools/utils.sh +++ b/tools/utils.sh @@ -53,321 +53,4 @@ die() { log_error "$*" stty echo exit 1 -} - -function trim_hostname() { - HOST_NAME=$(get_hostname) - local length=${#HOST_NAME} - if [ "$length" -eq 0 ]; then - random_string=$(head /dev/urandom | tr -dc 'a-z' | head -c 6) - echo "node-$random_string" - else - echo "$HOST_NAME" - fi -} - -function get_hostname() { - if [ "$(command -v hostname)" ]; then - hostname - elif [ "$(command -v python3)" ]; then - python3 -c "import socket; print(socket.gethostname())" - else - die "cannot get hostname, need \"hostname\" or \"python3\"" - fi -} - -function get_ip_list() { - if [ "$(command -v ip)" ]; then - ip addr show | grep inet | grep -vw "127.0.0.1" | grep -vw "172.17.0.1" | grep -v inet6 | awk '{print $2}' | cut -f1 -d '/' - elif [ "$(command -v ifconfig)" ]; then - ifconfig | grep inet | grep -vw "127.0.0.1" | grep -vw "172.17.0.1" | grep -v inet6 | awk '{print $2}' - elif [ "$(command -v python3)" ]; then - python3 -c "import socket; print('\n'.join(set(e[4][0] for e in socket.getaddrinfo(socket.gethostname(), None))));" - else - die "cannot get host ip, need \"ip\" or \"ifconfig\" or \"python3\"" - fi -} - -# get node IP automatically -function get_ip_auto() { - echo "Try to get IP address of this device" - ip_list_len=$(get_ip_list | wc -l) - local_ip=0 - if [ "$ip_list_len" -ge 2 ]; then - echo "Cannot get IP address of this device. Please choose the appropriate one manually" - for i in $(seq 1 "$ip_list_len"); do - ip=$(get_ip_list | head -n "$i" | tail -n 1) - read -rp "Local IP address is ${ip}. Press y to ensure or press Enter to skip:" conf - conf_flag='x'$conf - if [ "$conf_flag" == 'xy' ]; then - local_ip=$ip - break - fi - done - fi - if [ "$ip_list_len" -eq 1 ]; then - local_ip=$(get_ip_list) - fi -} - -function get_machine_ips() { - if [ "$(command -v ip)" ]; then - ip addr show | grep inet | grep -v inet6 | awk '{print $2}' | cut -f1 -d '/' - elif [ "$(command -v ifconfig)" ]; then - ifconfig | grep inet | grep -v inet6 | awk '{print $2}' | cut -f2 -d: - elif [ "$(command -v python3)" ]; then - python3 -c "import socket; print('\n'.join(set(e[4][0] for e in socket.getaddrinfo(socket.gethostname(), None))));" - else - die "cannot get host ip, need \"ip\" or \"ifconfig\" or \"python3\"" - fi -} - -function check_ip_exist() { - local ip_address=$1 - local machine_ips=$(get_machine_ips) - for machine_ip in ${machine_ips}; do - if [ "X${machine_ip}" == "X${ip_address}" ]; then - return 0 - fi - done - return 1 -} - -# please install libxml2 first -function init_config_var() { - local config_file=$1 - DEPLOY_PATH="$(xmllint --xpath "string(//config/deploy_path)" "${config_file}")" - LOCAL_IP="$(xmllint --xpath "string(//config/local_ip)" "${config_file}")" - REPO_PORT="$(xmllint --xpath "string(//config/function_repo_port)" "${config_file}")" - ADMIN_PORT="$(xmllint --xpath "string(//config/admin_port)" "${config_file}")" - export DEPLOY_PATH LOCAL_IP REPO_PORT ADMIN_PORT - - ETCD_AUTH_TYPE="$(xmllint --xpath "string(//config/etcd_auth_type)" "${config_file}")" - ETCD_PORT="$(xmllint --xpath "string(//config/etcd_port)" "${config_file}")" - ETCD_IP="$(xmllint --xpath "string(//config/etcd_ip)" "${config_file}")" - ETCD_PROXY_PORT="$(xmllint --xpath "string(//config/etcd_proxy_ports)" "${config_file}")" - ETCD_PROXY_ENABLE="$(xmllint --xpath "string(//config/etcd_proxy_enable)" "${config_file}")" - export ETCD_AUTH_TYPE ETCD_PORT ETCD_IP ETCD_PROXY_PORT ETCD_PROXY_ENABLE - - MINIO_IP="$(xmllint --xpath "string(//config/minio_ip)" "${config_file}")" - MINIO_PORT="$(xmllint --xpath "string(//config/minio_port)" "${config_file}")" - export MINIO_IP MINIO_PORT - - REDIS_PORT="$(xmllint --xpath "string(//config/redis_port)" "${config_file}")" - export REDIS_PORT - - WORKERMGR_LISTEN_PORT="$(xmllint --xpath "string(//config/workermgr_listen_port)" "${config_file}")" - CODE_DIR="$(xmllint --xpath "string(//config/code_dir)" "${config_file}")" - DS_MASTER_IP="$(xmllint --xpath "string(//config/ds_master_ip)" "${config_file}")" - DS_MASTER_PORT="$(xmllint --xpath "string(//config/ds_master_port)" "${config_file}")" - export WORKERMGR_LISTEN_PORT CODE_DIR DS_MASTER_IP DS_MASTER_PORT - - LOG_LEVEL="$(xmllint --xpath "string(//config/log/function_system/level)" "${config_file}")" - RUNTIME_LOG_LEVEL="$(xmllint --xpath "string(//config/log/runtime/level)" "${config_file}")" - export LOG_LEVEL RUNTIME_LOG_LEVEL - - FUNCTIONCORE_LOG_PATH="$(xmllint --xpath "string(//config/log/function_system/path)" "${config_file}")" - export FUNCTIONCORE_LOG_PATH - if [ "$FUNCTIONCORE_LOG_PATH" = "default" ]; then - unset FUNCTIONCORE_LOG_PATH - fi - - RUNTIME_LOG_PATH="$(xmllint --xpath "string(//config/log/runtime/path)" "${config_file}")" - export RUNTIME_LOG_PATH - if [ "$RUNTIME_LOG_PATH" = "default" ]; then - unset RUNTIME_LOG_PATH - fi - LOG_ROLLING_MAXSIZE="$(xmllint --xpath "string(//config/log/function_system/rolling/maxsize)" "${config_file}")" - LOG_ROLLING_MAXFILES="$(xmllint --xpath "string(//config/log/function_system/rolling/maxfiles)" "${config_file}")" - LOG_ASYNC_LOGBUFSECS="$(xmllint --xpath "string(//config/log/function_system/async/logBufSecs)" "${config_file}")" - LOG_ASYNC_MAXQUEUESIZE="$(xmllint --xpath "string(//config/log/function_system/async/maxQueueSize)" "${config_file}")" - LOG_ASYNC_THREADCOUNT="$(xmllint --xpath "string(//config/log/function_system/async/threadCount)" "${config_file}")" - LOG_ALSOLOGTOSTDERR="$(xmllint --xpath "string(//config/log/function_system/alsologtostderr)" "${config_file}")" - export LOG_ROLLING_MAXSIZE LOG_ROLLING_MAXFILES LOG_ASYNC_LOGBUFSECS LOG_ASYNC_MAXQUEUESIZE LOG_ASYNC_THREADCOUNT - export LOG_ALSOLOGTOSTDERR - - GLOBAL_SCHEDULER_PORT="$(xmllint --xpath "string(//config/global_scheduler_port)" "${config_file}")" - LOKI_IP="$(xmllint --xpath "string(//config/loki_ip)" "${config_file}")" - LOKI_PORT="$(xmllint --xpath "string(//config/loki_port)" "${config_file}")" - export GLOBAL_SCHEDULER_PORT LOKI_IP LOKI_PORT - - export SYS_FUNC_RETRY_PERIOD="$(xmllint --xpath "string(//config/sys_func_retry_period)" "${config_file}")" -} - -declare -A PORT_HASH_MAP - -function set_master_used_ds_port() { - for PORT in $MASTER_USED_DS_PORT; do - PORT_HASH_MAP["${PORT}"]="${PORT}" - done -} - -declare -a ORDERED_PORTS=() -function get_free_port() { - local bind_ip="$1" - local port_min="$2" - local port_max="$3" - command -v nc &>/dev/null - local ret=$? - # if nc is not install or nc disabled, skip port check - if [ ${ret} != 0 ] || [ "x${DISABLE_NC_CHECK}" = "xtrue" ]; then - PORT=$(shuf -i "${port_min}"-"${port_max}" -n 1) - while [[ ${PORT_HASH_MAP["$PORT"]} ]]; do - PORT=$(shuf -i "${port_min}"-"${port_max}" -n 1) - done - PORT_HASH_MAP[$PORT]=$PORT - ORDERED_PORTS=("$PORT" "${ORDERED_PORTS[@]}") - echo "$PORT" - return 0 - fi - - CHECK="port not assigned" - PORT="port" - while [[ -n "$CHECK" ]]; do - PORT=$(shuf -i "${port_min}"-"${port_max}" -n 1) - if [[ -z "${PORT_HASH_MAP[$PORT]}" ]]; then - set +e; CHECK=$(LD_LIBRARY_PATH="" timeout 0.2 nc -l "$bind_ip" "$PORT" >/dev/null 2>&1) - fi - done - PORT_HASH_MAP[$PORT]=$PORT - ORDERED_PORTS=("$PORT" "${ORDERED_PORTS[@]}") - echo "$PORT" - return 0 -} - -function check_port() { - command -v nc &>/dev/null - local ret=$? - # if nc is not install or nc disabled, skip port check - if [ ${ret} != 0 ] || [ "x${DISABLE_NC_CHECK}" = "xtrue" ]; then - return 0 - fi - local bind_ip="$1" - local port="$2" - local check=$(LD_LIBRARY_PATH="" timeout 0.2 nc -l "$bind_ip" "$port" >/dev/null 2>&1) - if [[ -n "$check" ]]; then - return 1 - fi - return 0 -} - -function log_backup() -{ - local log_file_dir="$1" - local log_file_num="$2" - local name_suffix="$3" - local node_id="$4" - local date_time old_file cur_log_file_num logfile_size - local maxsize=$((1024*1024*10)) - if [ ! -f "$log_file_dir/${node_id}${name_suffix}.log" ]; then - return - fi - logfile_size=$(stat -c %s "$log_file_dir/${node_id}${name_suffix}.log" | tr -d '\n') - - if [ "$logfile_size" -lt "$maxsize" ]; then - return - fi - - date_time=$(date +%Y_%m_%d_%H_%M_%S) - local compress_file_name="${node_id}${name_suffix}_${date_time}" - - cp "$log_file_dir/${node_id}${name_suffix}.log" "$log_file_dir/$compress_file_name" - echo '' > "$log_file_dir/${node_id}${name_suffix}.log" - tar -zcf "$log_file_dir/$compress_file_name".tar.gz -C "$log_file_dir" "${node_id}${name_suffix}_${date_time}" - rm -rf "$log_file_dir/$compress_file_name" - while :; do - cur_log_file_num=$(ls -l $log_file_dir/${node_id}${name_suffix}_*.tar.gz | wc -l) - if [ "$cur_log_file_num" -gt "$log_file_num" ];then - old_file=$(ls -tl $log_file_dir/${node_id}${name_suffix}_*.tar.gz | tail -1 | awk -F " " '{print $9}') - [[ -n "$old_file" ]] && rm -rf "$old_file" - else - break; - fi - done -} - -function set_runtime_pid_list() { - local ppid - for ppid in "${runtime_ppid_list[@]}"; do - child_process_pids=$(pgrep -P "$ppid") - for child_pid in $child_process_pids; do - runtime_pid_list+=("${child_pid}") - done - done -} - -function record_runtime_ppid() { - local parent_process_name - local pid=$1 - if [ "X${MERGE_PROCESS_ENABLE^^}" = "XTRUE" ];then - parent_process_name="function_agent" - else - parent_process_name="runtime_manager" - fi - process_name=$(ps -p $pid -o comm=) - if [ "${process_name}" == "${parent_process_name}" ]; then - runtime_ppid_list+=("$pid") - fi -} - -function record() { - local fd_threshold=4000 - local fd_num - local pid=$1 - fd_num=$(ls /proc/"$pid"/fd 2>/dev/null| wc -l) - if [ "$fd_num" == 0 ]; then - return - fi - log_info "$pid fd num: $fd_num" - if [ "$fd_num" -gt "$fd_threshold" ]; then - ls -l /proc/"$pid"/fd - fi - log_info "$pid stat: $(($(date '+%s')*1000+10#$(date '+%N')/1000000)) $(cat /proc/"$pid"/stat 2>/dev/null)" - ps -ef | awk -v pid="$pid" '{if(pid==$2)print}' - record_runtime_ppid $pid -} - -function monitor() { - local log_dir=$1 - local node_id=$2 - local compress_file_num=5 - local pgid pid - runtime_ppid_list=() - log_info "start to monitor" &>>"${log_dir}"/${node_id}-monitor_std.log - pgid=$(ps -o pgid $$|tail -n 1|sed s/[[:space:]]//g) - for pid in $(ps axj | awk -v pgid="$pgid" '{if(pgid==$3)print}' | grep -v grep |grep -v PPID | awk '{print $2}'); - do - record $pid &>>"${log_dir}"/${node_id}-monitor_std.log - done - - runtime_pid_list=() - set_runtime_pid_list - for pid in "${runtime_pid_list[@]}"; do - record $pid &>>"${log_dir}"/${node_id}-monitor_std.log - done - log_backup "${log_dir}" "$compress_file_num" "-monitor_std" "${node_id}" -} - -function check_and_set_component_checklist() { - local new_component=$1 - local pid=$2 - pid_table["${new_component}"]=${pid} - for c in ${COMPONENT_CHECKLIST[@]}; do - if [ X"${c}" = X"${new_component}" ]; then - return - fi - done - COMPONENT_CHECKLIST+=("${new_component}") -} - -function show_component_error_log() { - local log_name=$1 - local key_word=$2 - local note=$3 - local output=$(grep -E "^[EW]" ${log_name} | grep "${key_word}") - if [ -n "$output" ]; then - log_warning "${note} occurs error, errors as follows:" - grep -E "^[EW]" ${log_name} | grep "${key_word}" | xargs -I {} echo "{}" - log_warning "For more details, see ${log_name}." - fi } \ No newline at end of file -- Gitee From b389c362632dd28e4bb855cbe41164e57e18d601 Mon Sep 17 00:00:00 2001 From: Chamberlain Date: Thu, 20 Nov 2025 15:40:03 +0800 Subject: [PATCH 3/3] remove unity --- CMakeLists.txt | 83 +++++++++---------- functionsystem/src/common/CMakeLists.txt | 3 - functionsystem/src/common/aksk/CMakeLists.txt | 1 - functionsystem/src/common/aksk/aksk_util.cpp | 2 +- .../src/common/common_flags/CMakeLists.txt | 1 - .../src/common/explorer/CMakeLists.txt | 1 - .../src/common/file_monitor/CMakeLists.txt | 1 - .../file_monitor/monitor_callback_actor.cpp | 5 ++ .../src/common/heartbeat/CMakeLists.txt | 1 - functionsystem/src/common/hex/CMakeLists.txt | 1 - functionsystem/src/common/http/CMakeLists.txt | 1 - .../src/common/kube_client/CMakeLists.txt | 1 - .../src/common/leader/CMakeLists.txt | 1 - .../common/meta_store_adapter/CMakeLists.txt | 1 - .../src/common/metrics/CMakeLists.txt | 1 - .../src/common/network/CMakeLists.txt | 1 - .../src/common/profile/CMakeLists.txt | 1 - .../src/common/proto/CMakeLists.txt | 2 - .../src/common/register/CMakeLists.txt | 1 - .../src/common/resource_lock/CMakeLists.txt | 1 - .../src/common/resource_view/CMakeLists.txt | 1 - functionsystem/src/common/rpc/CMakeLists.txt | 1 - .../common/scheduler_topology/CMakeLists.txt | 1 - .../src/common/service_json/CMakeLists.txt | 1 - .../src/common/status/CMakeLists.txt | 1 - .../src/common/trace/CMakeLists.txt | 1 - .../src/common/utils/time_utils.cpp | 36 ++++---- functionsystem/src/common/utils/time_utils.h | 21 ++++- .../src/common/yaml_tool/CMakeLists.txt | 1 - .../src/domain_scheduler/CMakeLists.txt | 1 - .../src/function_agent/CMakeLists.txt | 1 - .../code_deployer/remote_deployer.cpp | 1 + .../code_deployer/s3_deployer.cpp | 1 + .../common/static_function_util.cpp | 2 + .../src/function_agent/common/utils.cpp | 1 + .../driver/function_agent_driver.cpp | 1 + .../src/function_master/common/CMakeLists.txt | 1 - .../global_scheduler/CMakeLists.txt | 1 - .../instance_manager/CMakeLists.txt | 1 - .../resource_group_manager/CMakeLists.txt | 1 - .../src/function_master/scaler/CMakeLists.txt | 1 - .../system_function_pod_manager.h | 2 + .../system_function_loader/CMakeLists.txt | 1 - .../bootstrap_actor.cpp | 10 --- .../system_function_loader/bootstrap_actor.h | 9 ++ .../function_proxy/busproxy/CMakeLists.txt | 2 - .../src/function_proxy/common/CMakeLists.txt | 1 - .../local_scheduler/CMakeLists.txt | 2 - .../ds_healthy_checker/ds_healthy_checker.cpp | 2 + functionsystem/src/iam_server/CMakeLists.txt | 1 - .../src/meta_store/client/cpp/CMakeLists.txt | 1 - .../meta_store_client_mgr.cpp | 1 + .../src/meta_store/server/src/CMakeLists.txt | 1 - .../src/runtime_manager/CMakeLists.txt | 1 - .../src/runtime_manager/config/flags.cpp | 1 + 55 files changed, 102 insertions(+), 120 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 189d90a..3d25e26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,48 +15,41 @@ cmake_minimum_required(VERSION 3.16.1) project(kernel LANGUAGES CXX C) -message(STATUS "kernel PROJECT_SOURCE_DIR：${PROJECT_SOURCE_DIR}") - -# Check Git submodule status (only active in Git repositories) -execute_process( - COMMAND git rev-parse --is-inside-work-tree - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE IS_GIT_REPO - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -if(IS_GIT_REPO) - # Check submodule status recursively - execute_process( - COMMAND git submodule status --recursive - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE SUBMODULE_STATUS - RESULT_VARIABLE GIT_SUBMOD_RESULT - ERROR_QUIET - ) - - if(GIT_SUBMOD_RESULT EQUAL 0) - # Detect uninitialized submodules (lines starting with '-') - string(REGEX MATCH "(\n|^)-" UNINITIALIZED "${SUBMODULE_STATUS}") - if(UNINITIALIZED) - message(STATUS "Found uninitialized submodules. Initializing...") - - # Automatically initialize and update submodules - execute_process( - COMMAND git submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_UPDATE_RESULT - ) - - if(NOT GIT_SUBMOD_UPDATE_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to initialize submodules. Error code: ${GIT_SUBMOD_UPDATE_RESULT}") - endif() - - message(STATUS "Submodules initialized successfully") - endif() - endif() -endif() - -add_subdirectory(functionsystem) -add_subdirectory(runtime) \ No newline at end of file +include_directories(common/litebus/include) +include_directories(common/logs/include) +include_directories(common/meta_store/client/cpp/include) +include_directories(common/meta_store/server/cpp/include) +include_directories(common/utils/) +include_directories(common/utils/proto/pb/) +include_directories(common/utils/metadata) +include_directories(posix/) +include_directories(posix/proto/pb) +include_directories(functionsystem/src/) +include_directories(functionsystem/src/common) +include_directories(functionsystem/src/domain_scheduler) +include_directories(functionsystem/src/function_agent) +include_directories(functionsystem/src/function_proxy) +include_directories(functionsystem/src/function_proxy/local_scheduler/) +include_directories(functionsystem/src/function_master) + + +include_directories(vendor/out/openEuler/Install/cjson/include) +include_directories(vendor/out/openEuler/Install/grpc/include) +include_directories(vendor/out/openEuler/Install/gtest_1_10_0/include) +include_directories(vendor/out/openEuler/Install/gtest_1_12_1/include) +include_directories(vendor/out/openEuler/Install/protobuf/include) +include_directories(vendor/out/openEuler/Install/spdlog/include) + +file(GLOB_RECURSE SOURCES1 functionsystem/src/*.cpp) +file(GLOB_RECURSE SOURCES2 functionsystem/src/*.cc) +file(GLOB_RECURSE SOURCES3 functionsystem/src/*.h) +file(GLOB_RECURSE SOURCES4 common/litebus/src/*.cpp) +file(GLOB_RECURSE SOURCES5 common/litebus/src/*.hpp) +file(GLOB_RECURSE SOURCES6 common/metrics/src/*.h) +file(GLOB_RECURSE SOURCES7 common/metrics/src/*.cpp) +file(GLOB_RECURSE SOURCES8 common/meta_store/client/src/*.cpp) +file(GLOB_RECURSE SOURCES9 common/meta_store/client/src/*.h) +file(GLOB_RECURSE SOURCES10 common/meta_store/server/src/*.cpp) +file(GLOB_RECURSE SOURCES11 common/meta_store/server/src/*.h) + +add_executable(main ${SOURCES1} ${SOURCES2} ${SOURCES3} ${SOURCES4} ${SOURCES5} ${SOURCES6} ${SOURCES7} ${SOURCES8} ${SOURCES9} ${SOURCES10} ${SOURCES11}) \ No newline at end of file diff --git a/functionsystem/src/common/CMakeLists.txt b/functionsystem/src/common/CMakeLists.txt index 6b05377..e9e324e 100644 --- a/functionsystem/src/common/CMakeLists.txt +++ b/functionsystem/src/common/CMakeLists.txt @@ -49,14 +49,12 @@ add_subdirectory(resource_lock) add_library(utils STATIC "") -set_target_properties(utils PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/utils FS_UTILS_SRCS) add_dependencies(utils status) target_sources(utils PRIVATE ${FS_UTILS_SRCS}) target_link_libraries(utils PUBLIC status ${crypto_LIB} ${ssl_LIB}) add_library(scheduler STATIC "") -set_target_properties(scheduler PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/scheduler SCHEDULER_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/schedule_decision SCHEDULER_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/schedule_decision/scheduler SCHEDULER_SRCS) @@ -73,7 +71,6 @@ target_sources(scheduler PRIVATE ${SCHEDULER_SRCS}) target_link_libraries(scheduler PUBLIC status posix_pb) add_library(schedule_plugin_common STATIC "") -set_target_properties(schedule_plugin_common PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/schedule_plugin/common PLUGIN_COMMON_SRCS) add_dependencies(schedule_plugin_common status posix_pb resource_view) target_sources(schedule_plugin_common PRIVATE ${PLUGIN_COMMON_SRCS}) diff --git a/functionsystem/src/common/aksk/CMakeLists.txt b/functionsystem/src/common/aksk/CMakeLists.txt index 1b410c3..6f12eb3 100644 --- a/functionsystem/src/common/aksk/CMakeLists.txt +++ b/functionsystem/src/common/aksk/CMakeLists.txt @@ -19,7 +19,6 @@ add_library(aksk STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(aksk PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} AKSK_SRCS) add_dependencies(aksk hex http) target_sources(aksk PRIVATE ${AKSK_SRCS}) diff --git a/functionsystem/src/common/aksk/aksk_util.cpp b/functionsystem/src/common/aksk/aksk_util.cpp index 58330ed..53c741c 100644 --- a/functionsystem/src/common/aksk/aksk_util.cpp +++ b/functionsystem/src/common/aksk/aksk_util.cpp @@ -23,7 +23,7 @@ #include "common/hex/hex.h" #include "common/http/http_util.h" #include "common/proto/pb/posix_pb.h" -#include "common/utils/time_utils.h" +#include "utils/time_utils.h" #include "openssl/sha.h" #include "sign_request.h" #include "utils/os_utils.hpp" diff --git a/functionsystem/src/common/common_flags/CMakeLists.txt b/functionsystem/src/common/common_flags/CMakeLists.txt index 299db95..c4b3727 100644 --- a/functionsystem/src/common/common_flags/CMakeLists.txt +++ b/functionsystem/src/common/common_flags/CMakeLists.txt @@ -19,7 +19,6 @@ add_library(common_flags STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(common_flags PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} COMMON_FLAGS_SRCS) target_sources(common_flags PRIVATE ${COMMON_FLAGS_SRCS}) target_link_libraries(common_flags PUBLIC ${litebus_ALL_LIB}) \ No newline at end of file diff --git a/functionsystem/src/common/explorer/CMakeLists.txt b/functionsystem/src/common/explorer/CMakeLists.txt index b85664e..5c42c2e 100644 --- a/functionsystem/src/common/explorer/CMakeLists.txt +++ b/functionsystem/src/common/explorer/CMakeLists.txt @@ -14,7 +14,6 @@ project(explore) add_library(explorer STATIC "") -set_target_properties(explorer PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} EXPLORER_SRCS) add_dependencies(explorer status meta_store_client resource_lock) target_sources(explorer PRIVATE ${EXPLORER_SRCS}) diff --git a/functionsystem/src/common/file_monitor/CMakeLists.txt b/functionsystem/src/common/file_monitor/CMakeLists.txt index 1e9beb2..6fe78af 100644 --- a/functionsystem/src/common/file_monitor/CMakeLists.txt +++ b/functionsystem/src/common/file_monitor/CMakeLists.txt @@ -14,7 +14,6 @@ project(file_monitor) add_library(file_monitor STATIC "") -set_target_properties(file_monitor PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} FILE_MONITOR_SRCS) add_dependencies(file_monitor status) target_sources(file_monitor PRIVATE ${FILE_MONITOR_SRCS}) diff --git a/functionsystem/src/common/file_monitor/monitor_callback_actor.cpp b/functionsystem/src/common/file_monitor/monitor_callback_actor.cpp index e4cd776..4d99aae 100644 --- a/functionsystem/src/common/file_monitor/monitor_callback_actor.cpp +++ b/functionsystem/src/common/file_monitor/monitor_callback_actor.cpp @@ -14,6 +14,11 @@ * limitations under the License. */ #include "common/file_monitor/monitor_callback_actor.h" +#include "async/asyncafter.hpp" +#include "utils/os_utils.hpp" +#include "utils/string_utils.hpp" +#include "common/utils/files.h" +#include "common/utils/exec_utils.h" #include "common/constants/constants.h" diff --git a/functionsystem/src/common/heartbeat/CMakeLists.txt b/functionsystem/src/common/heartbeat/CMakeLists.txt index 3734e0d..abd436b 100644 --- a/functionsystem/src/common/heartbeat/CMakeLists.txt +++ b/functionsystem/src/common/heartbeat/CMakeLists.txt @@ -16,6 +16,5 @@ project(heartbeat) add_library(heartbeat STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(heartbeat PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} HEARTBEAT_SRCS) target_sources(heartbeat PRIVATE ${HEARTBEAT_SRCS}) \ No newline at end of file diff --git a/functionsystem/src/common/hex/CMakeLists.txt b/functionsystem/src/common/hex/CMakeLists.txt index a79ec57..c035741 100644 --- a/functionsystem/src/common/hex/CMakeLists.txt +++ b/functionsystem/src/common/hex/CMakeLists.txt @@ -14,7 +14,6 @@ project(hex) add_library(hex STATIC "") -set_target_properties(hex PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} HEX_SRCS) add_dependencies(hex status) target_sources(hex PRIVATE ${HEX_SRCS}) diff --git a/functionsystem/src/common/http/CMakeLists.txt b/functionsystem/src/common/http/CMakeLists.txt index 9e3d570..59277aa 100644 --- a/functionsystem/src/common/http/CMakeLists.txt +++ b/functionsystem/src/common/http/CMakeLists.txt @@ -16,7 +16,6 @@ project(http) add_library(http STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(http PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} HTTP_SRCS) add_dependencies(http status) target_sources(http PRIVATE ${HTTP_SRCS}) diff --git a/functionsystem/src/common/kube_client/CMakeLists.txt b/functionsystem/src/common/kube_client/CMakeLists.txt index 051b0b8..ca3651e 100644 --- a/functionsystem/src/common/kube_client/CMakeLists.txt +++ b/functionsystem/src/common/kube_client/CMakeLists.txt @@ -14,7 +14,6 @@ project(kube_client) add_library(kube_client STATIC "") -set_target_properties(kube_client PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} KUBE_CLIENT_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/api KUBE_CLIENT_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/health_monitor KUBE_CLIENT_SRCS) diff --git a/functionsystem/src/common/leader/CMakeLists.txt b/functionsystem/src/common/leader/CMakeLists.txt index 9a6368f..f059fab 100644 --- a/functionsystem/src/common/leader/CMakeLists.txt +++ b/functionsystem/src/common/leader/CMakeLists.txt @@ -14,7 +14,6 @@ project(leader) add_library(leader STATIC "") -set_target_properties(leader PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} LEADER_SRCS) add_dependencies(leader status meta_store_client explorer) target_sources(leader PRIVATE ${LEADER_SRCS}) diff --git a/functionsystem/src/common/meta_store_adapter/CMakeLists.txt b/functionsystem/src/common/meta_store_adapter/CMakeLists.txt index ef171d5..d8b099d 100644 --- a/functionsystem/src/common/meta_store_adapter/CMakeLists.txt +++ b/functionsystem/src/common/meta_store_adapter/CMakeLists.txt @@ -14,7 +14,6 @@ project(meta_store_adapter) add_library(meta_store_adapter STATIC "") -set_target_properties(meta_store_adapter PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} META_STORE_ADAPTER_SRCS) add_dependencies(meta_store_adapter posix_pb resource_view) target_sources(meta_store_adapter PRIVATE ${META_STORE_ADAPTER_SRCS}) diff --git a/functionsystem/src/common/metrics/CMakeLists.txt b/functionsystem/src/common/metrics/CMakeLists.txt index c7dd86f..90fd287 100644 --- a/functionsystem/src/common/metrics/CMakeLists.txt +++ b/functionsystem/src/common/metrics/CMakeLists.txt @@ -22,7 +22,6 @@ include_directories(${UTILS_INCLUDE}) include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(metrics_adapter STATIC "") -set_target_properties(metrics_adapter PROPERTIES UNITY_BUILD ON) file(GLOB_RECURSE METRICS_ADAPTER_SRCS *.cpp) diff --git a/functionsystem/src/common/network/CMakeLists.txt b/functionsystem/src/common/network/CMakeLists.txt index 4b4c18f..5a39757 100644 --- a/functionsystem/src/common/network/CMakeLists.txt +++ b/functionsystem/src/common/network/CMakeLists.txt @@ -14,6 +14,5 @@ project(network) add_library(network STATIC "") -set_target_properties(network PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} NETWORK_SRCS) target_sources(network PRIVATE ${NETWORK_SRCS}) \ No newline at end of file diff --git a/functionsystem/src/common/profile/CMakeLists.txt b/functionsystem/src/common/profile/CMakeLists.txt index cc6c0a0..f5e282e 100644 --- a/functionsystem/src/common/profile/CMakeLists.txt +++ b/functionsystem/src/common/profile/CMakeLists.txt @@ -14,7 +14,6 @@ project(profile) add_library(profile STATIC "") -set_target_properties(profile PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} PROFILE_SRCS) add_dependencies(profile posix_pb) target_sources(profile PRIVATE ${PROFILE_SRCS}) diff --git a/functionsystem/src/common/proto/CMakeLists.txt b/functionsystem/src/common/proto/CMakeLists.txt index 6d9d4fd..c447d6f 100644 --- a/functionsystem/src/common/proto/CMakeLists.txt +++ b/functionsystem/src/common/proto/CMakeLists.txt @@ -27,8 +27,6 @@ GENERATE_GRPC_CPP(GRPCPB_SRCS GRPCPB_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/pb/posix SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/posix) include_directories(${CMAKE_CURRENT_LIST_DIR}/pb) add_library(posix_pb STATIC ${GRPCPB_SRCS}) -set_target_properties(posix_pb PROPERTIES UNITY_BUILD ON) -set_target_properties(posix_pb PROPERTIES UNITY_BUILD_BATCH_SIZE 2) add_dependencies(posix_pb protobuf grpc) target_compile_options(posix_pb PRIVATE "-fvisibility=hidden") target_link_libraries(posix_pb PUBLIC diff --git a/functionsystem/src/common/register/CMakeLists.txt b/functionsystem/src/common/register/CMakeLists.txt index 49cb923..640756e 100644 --- a/functionsystem/src/common/register/CMakeLists.txt +++ b/functionsystem/src/common/register/CMakeLists.txt @@ -14,7 +14,6 @@ project(register) add_library(register STATIC "") -set_target_properties(register PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} REGISTER_SRCS) add_dependencies(register status heartbeat) target_sources(register PRIVATE ${REGISTER_SRCS}) diff --git a/functionsystem/src/common/resource_lock/CMakeLists.txt b/functionsystem/src/common/resource_lock/CMakeLists.txt index f4f7d9e..a140fd5 100644 --- a/functionsystem/src/common/resource_lock/CMakeLists.txt +++ b/functionsystem/src/common/resource_lock/CMakeLists.txt @@ -14,7 +14,6 @@ project(resource_lock) add_library(resource_lock STATIC "") -set_target_properties(resource_lock PROPERTIES UNITY_BUILD ON) add_dependencies(resource_lock kube_client) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} RESOURCE_LOCK_SRCS) target_sources(resource_lock PRIVATE ${RESOURCE_LOCK_SRCS}) diff --git a/functionsystem/src/common/resource_view/CMakeLists.txt b/functionsystem/src/common/resource_view/CMakeLists.txt index 2b7c789..dc26604 100644 --- a/functionsystem/src/common/resource_view/CMakeLists.txt +++ b/functionsystem/src/common/resource_view/CMakeLists.txt @@ -14,7 +14,6 @@ project(resource_view) add_library(resource_view STATIC "") -set_target_properties(resource_view PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} RESOURCE_VIEW_SRCS) add_dependencies(resource_view posix_pb) target_sources(resource_view PRIVATE ${RESOURCE_VIEW_SRCS}) diff --git a/functionsystem/src/common/rpc/CMakeLists.txt b/functionsystem/src/common/rpc/CMakeLists.txt index c45fee4..1ed5c2c 100644 --- a/functionsystem/src/common/rpc/CMakeLists.txt +++ b/functionsystem/src/common/rpc/CMakeLists.txt @@ -20,7 +20,6 @@ add_library(rpc STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(rpc PROPERTIES UNITY_BUILD ON) file(GLOB_RECURSE RPC_SRCS *.cpp) target_sources(rpc PRIVATE ${RPC_SRCS}) target_compile_options(rpc PRIVATE "-fvisibility=hidden") diff --git a/functionsystem/src/common/scheduler_topology/CMakeLists.txt b/functionsystem/src/common/scheduler_topology/CMakeLists.txt index 7bc8c0c..d1806c1 100644 --- a/functionsystem/src/common/scheduler_topology/CMakeLists.txt +++ b/functionsystem/src/common/scheduler_topology/CMakeLists.txt @@ -14,7 +14,6 @@ project(scheduler_topology) add_library(scheduler_topology STATIC "") -set_target_properties(scheduler_topology PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} TOPOLOGY_SRCS) add_dependencies(scheduler_topology posix_pb) target_sources(scheduler_topology PRIVATE ${TOPOLOGY_SRCS}) diff --git a/functionsystem/src/common/service_json/CMakeLists.txt b/functionsystem/src/common/service_json/CMakeLists.txt index d58f3bd..606d46d 100644 --- a/functionsystem/src/common/service_json/CMakeLists.txt +++ b/functionsystem/src/common/service_json/CMakeLists.txt @@ -14,7 +14,6 @@ project(service_json) add_library(service_json STATIC "") -set_target_properties(service_json PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} SERVICE_JSON_SRCS) add_dependencies(service_json posix_pb resource_view) target_sources(service_json PRIVATE ${SERVICE_JSON_SRCS}) diff --git a/functionsystem/src/common/status/CMakeLists.txt b/functionsystem/src/common/status/CMakeLists.txt index b97168a..aeeb68a 100644 --- a/functionsystem/src/common/status/CMakeLists.txt +++ b/functionsystem/src/common/status/CMakeLists.txt @@ -19,7 +19,6 @@ add_library(status STATIC "") get_property(UTILS_INCLUDE GLOBAL PROPERTY "UTILS_INCLUDE") include_directories(${UTILS_INCLUDE}) -set_target_properties(status PROPERTIES UNITY_BUILD ON) aux_source_directory(${CMAKE_CURRENT_LIST_DIR} STATUS_SRCS) add_dependencies(status posix_pb) target_sources(status PRIVATE ${STATUS_SRCS}) diff --git a/functionsystem/src/common/trace/CMakeLists.txt b/functionsystem/src/common/trace/CMakeLists.txt index 8d53ed5..93d40b4 100644 --- a/functionsystem/src/common/trace/CMakeLists.txt +++ b/functionsystem/src/common/trace/CMakeLists.txt @@ -14,7 +14,6 @@ project(trace_adapter) add_library(trace_adapter STATIC "") -set_target_properties(trace_adapter PROPERTIES UNITY_BUILD ON) message(STATUS "building with observability") aux_source_directory(${CMAKE_CURRENT_LIST_DIR} TRACE_SRCS) add_dependencies(trace_adapter status) diff --git a/functionsystem/src/common/utils/time_utils.cpp b/functionsystem/src/common/utils/time_utils.cpp index 34a62ca..92c606b 100644 --- a/functionsystem/src/common/utils/time_utils.cpp +++ b/functionsystem/src/common/utils/time_utils.cpp @@ -17,23 +17,23 @@ #include "time_utils.h" namespace functionsystem { -std::tm ParseTimestamp(const std::string ×tamp) -{ - std::tm tm = {}; - std::istringstream ss(timestamp); - ss >> std::get_time(&tm, "%Y%m%dT%H%M%SZ"); - if (ss.fail()) { - return {}; - } - return tm; -} +// std::tm ParseTimestamp(const std::string ×tamp) +// { +// std::tm tm = {}; +// std::istringstream ss(timestamp); +// ss >> std::get_time(&tm, "%Y%m%dT%H%M%SZ"); +// if (ss.fail()) { +// return {}; +// } +// return tm; +// } -bool IsLaterThan(const std::string ×tamp1, const std::string ×tamp2, double seconds) -{ - std::tm tm1 = ParseTimestamp(timestamp1); - std::tm tm2 = ParseTimestamp(timestamp2); - std::time_t time1 = std::mktime(&tm1); - std::time_t time2 = std::mktime(&tm2); - return std::difftime(time1, time2) > seconds; -} +// bool IsLaterThan(const std::string ×tamp1, const std::string ×tamp2, double seconds) +// { +// std::tm tm1 = ParseTimestamp(timestamp1); +// std::tm tm2 = ParseTimestamp(timestamp2); +// std::time_t time1 = std::mktime(&tm1); +// std::time_t time2 = std::mktime(&tm2); +// return std::difftime(time1, time2) > seconds; +// } } // namespace functionsystem \ No newline at end of file diff --git a/functionsystem/src/common/utils/time_utils.h b/functionsystem/src/common/utils/time_utils.h index 87a82c1..78d0cf6 100644 --- a/functionsystem/src/common/utils/time_utils.h +++ b/functionsystem/src/common/utils/time_utils.h @@ -25,7 +25,24 @@ #include namespace functionsystem { -std::tm ParseTimestamp(const std::string ×tamp); -bool IsLaterThan(const std::string ×tamp1, const std::string ×tamp2, double seconds); +inline std::tm ParseTimestamp(const std::string ×tamp) +{ + std::tm tm = {}; + std::istringstream ss(timestamp); + ss >> std::get_time(&tm, "%Y%m%dT%H%M%SZ"); + if (ss.fail()) { + return {}; + } + return tm; +} +// bool IsLaterThan(const std::string ×tamp1, const std::string ×tamp2, double seconds); +inline bool IsLaterThan(const std::string ×tamp1, const std::string ×tamp2, double seconds) +{ + std::tm tm1 = ParseTimestamp(timestamp1); + std::tm tm2 = ParseTimestamp(timestamp2); + std::time_t time1 = std::mktime(&tm1); + std::time_t time2 = std::mktime(&tm2); + return std::difftime(time1, time2) > seconds; +} } // namespace functionsystem #endif // COMMON_UTILS_TIME_H diff --git a/functionsystem/src/common/yaml_tool/CMakeLists.txt b/functionsystem/src/common/yaml_tool/CMakeLists.txt index 219e6cc..f314804 100644 --- a/functionsystem/src/common/yaml_tool/CMakeLists.txt +++ b/functionsystem/src/common/yaml_tool/CMakeLists.txt @@ -3,7 +3,6 @@ project(yaml_tool) add_library(yaml_tool SHARED yaml_tool.cpp ) -set_target_properties(yaml_tool PROPERTIES UNITY_BUILD ON) add_dependencies(yaml_tool yaml) target_link_libraries(yaml_tool ${yaml_LIB}) diff --git a/functionsystem/src/domain_scheduler/CMakeLists.txt b/functionsystem/src/domain_scheduler/CMakeLists.txt index 8b931ad..47c504f 100644 --- a/functionsystem/src/domain_scheduler/CMakeLists.txt +++ b/functionsystem/src/domain_scheduler/CMakeLists.txt @@ -19,7 +19,6 @@ message(STATUS "Begin to configure domain_scheduler") set_property(GLOBAL PROPERTY SRC_DOMAIN_SCHEDULER_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}") add_library(domain_scheduler_lib STATIC domain_scheduler_launcher.cpp) -set_target_properties(domain_scheduler_lib PROPERTIES UNITY_BUILD ON) target_compile_options(domain_scheduler_lib PRIVATE -fPIE) add_subdirectory(startup) diff --git a/functionsystem/src/function_agent/CMakeLists.txt b/functionsystem/src/function_agent/CMakeLists.txt index 2186049..84ae7e3 100644 --- a/functionsystem/src/function_agent/CMakeLists.txt +++ b/functionsystem/src/function_agent/CMakeLists.txt @@ -30,7 +30,6 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/network FUNCTION_AGENT_SRCS) add_library(${FUNCTION_AGENT_LIB} STATIC # for [function_agent_lib] ${FUNCTION_AGENT_SRCS}) -set_target_properties(${FUNCTION_AGENT_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${FUNCTION_AGENT_LIB} PRIVATE -fPIE) add_dependencies(${FUNCTION_AGENT_LIB} zlib rpc) diff --git a/functionsystem/src/function_agent/code_deployer/remote_deployer.cpp b/functionsystem/src/function_agent/code_deployer/remote_deployer.cpp index 66be988..2819bef 100644 --- a/functionsystem/src/function_agent/code_deployer/remote_deployer.cpp +++ b/functionsystem/src/function_agent/code_deployer/remote_deployer.cpp @@ -27,6 +27,7 @@ #include "common/utils/ssl_config.h" #include "minizip/unzip.h" #include "utils/os_utils.hpp" +#include "common/constants.h" namespace functionsystem::function_agent { diff --git a/functionsystem/src/function_agent/code_deployer/s3_deployer.cpp b/functionsystem/src/function_agent/code_deployer/s3_deployer.cpp index 9faed99..826d2a5 100644 --- a/functionsystem/src/function_agent/code_deployer/s3_deployer.cpp +++ b/functionsystem/src/function_agent/code_deployer/s3_deployer.cpp @@ -24,6 +24,7 @@ #include "common/logs/logging.h" #include "common/utils/ssl_config.h" #include "minizip/unzip.h" +#include "common/utils/param_check.h" namespace functionsystem::function_agent { const uint32_t RECOVER_RETRY_COUNT = 3; diff --git a/functionsystem/src/function_agent/common/static_function_util.cpp b/functionsystem/src/function_agent/common/static_function_util.cpp index 16f631c..06dacaf 100644 --- a/functionsystem/src/function_agent/common/static_function_util.cpp +++ b/functionsystem/src/function_agent/common/static_function_util.cpp @@ -17,6 +17,8 @@ #include "static_function_util.h" +#include "common/utils/struct_transfer.h" + namespace functionsystem::function_agent { const std::string CONCURRENT_NUM_KEY = "ConcurrentNum"; const std::string CALL_TIMEOUT_KEY = "call_timeout"; diff --git a/functionsystem/src/function_agent/common/utils.cpp b/functionsystem/src/function_agent/common/utils.cpp index d42594a..99cd76b 100644 --- a/functionsystem/src/function_agent/common/utils.cpp +++ b/functionsystem/src/function_agent/common/utils.cpp @@ -20,6 +20,7 @@ #include #include +#include "constants.h" #include "common/aksk/aksk_util.h" #include "common/constants/constants.h" #include "common/create_agent_decision/create_agent_decision.h" diff --git a/functionsystem/src/function_agent/driver/function_agent_driver.cpp b/functionsystem/src/function_agent/driver/function_agent_driver.cpp index fb60c0f..a9c3363 100644 --- a/functionsystem/src/function_agent/driver/function_agent_driver.cpp +++ b/functionsystem/src/function_agent/driver/function_agent_driver.cpp @@ -21,6 +21,7 @@ #include "agent_service_actor.h" #include "async/future.hpp" +#include "code_deployer/shared_dir_deployer.h" #include "common/constants/actor_name.h" #include "common/logs/logging.h" #include "common/register/register_helper.h" diff --git a/functionsystem/src/function_master/common/CMakeLists.txt b/functionsystem/src/function_master/common/CMakeLists.txt index 1c4746d..0b1e0e7 100644 --- a/functionsystem/src/function_master/common/CMakeLists.txt +++ b/functionsystem/src/function_master/common/CMakeLists.txt @@ -10,7 +10,6 @@ add_library(${MASTER_COMMON_LIB} STATIC ${FLAGS_SRCS} ${BUSINESS_POLICY_SRCS} ) -set_target_properties(${MASTER_COMMON_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${MASTER_COMMON_LIB} PRIVATE -fPIE) target_include_directories(${MASTER_COMMON_LIB} PRIVATE ${CMAKE_CURRENT_LIST_DIR}) diff --git a/functionsystem/src/function_master/global_scheduler/CMakeLists.txt b/functionsystem/src/function_master/global_scheduler/CMakeLists.txt index 38c4015..7eab6ca 100644 --- a/functionsystem/src/function_master/global_scheduler/CMakeLists.txt +++ b/functionsystem/src/function_master/global_scheduler/CMakeLists.txt @@ -9,7 +9,6 @@ set_property(GLOBAL PROPERTY GLOBAL_SCHEDULER_LIB ${GLOBAL_SCHEDULER_LIB}) add_library(${GLOBAL_SCHEDULER_LIB} STATIC ${GLOBAL_SCHEDULER_SRCS} ${GLOBAL_SCHEDULER_MANAGER_SRCS}) -set_target_properties(${GLOBAL_SCHEDULER_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${GLOBAL_SCHEDULER_LIB} PRIVATE -fPIE) target_include_directories(${GLOBAL_SCHEDULER_LIB} PRIVATE ${CMAKE_CURRENT_LIST_DIR}) diff --git a/functionsystem/src/function_master/instance_manager/CMakeLists.txt b/functionsystem/src/function_master/instance_manager/CMakeLists.txt index b519b68..b76379a 100644 --- a/functionsystem/src/function_master/instance_manager/CMakeLists.txt +++ b/functionsystem/src/function_master/instance_manager/CMakeLists.txt @@ -23,7 +23,6 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(${INSTANCE_MANAGER_LIB} STATIC ${INSTANCE_MANAGER_SRCS}) -set_target_properties(${INSTANCE_MANAGER_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${INSTANCE_MANAGER_LIB} PRIVATE -fPIE) target_include_directories(${INSTANCE_MANAGER_LIB} PRIVATE diff --git a/functionsystem/src/function_master/resource_group_manager/CMakeLists.txt b/functionsystem/src/function_master/resource_group_manager/CMakeLists.txt index f1bcfae..7cd13a0 100644 --- a/functionsystem/src/function_master/resource_group_manager/CMakeLists.txt +++ b/functionsystem/src/function_master/resource_group_manager/CMakeLists.txt @@ -23,7 +23,6 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(${RESOURCE_GROUP_MANAGER_LIB} STATIC ${RESOURCE_GROUP_MANAGER_SRCS}) -set_target_properties(${RESOURCE_GROUP_MANAGER_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${RESOURCE_GROUP_MANAGER_LIB} PRIVATE -fPIE) target_include_directories(${RESOURCE_GROUP_MANAGER_LIB} PRIVATE diff --git a/functionsystem/src/function_master/scaler/CMakeLists.txt b/functionsystem/src/function_master/scaler/CMakeLists.txt index 9260501..713c87b 100644 --- a/functionsystem/src/function_master/scaler/CMakeLists.txt +++ b/functionsystem/src/function_master/scaler/CMakeLists.txt @@ -19,7 +19,6 @@ add_library(${SCALER_LIB} STATIC ${SCALER_UTILS_SRCS} ${SCALER_POOL_SRCS} ${SCALER_SRCS}) -set_target_properties(${SCALER_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${SCALER_LIB} PRIVATE -fPIE) target_include_directories(${SCALER_LIB} PRIVATE diff --git a/functionsystem/src/function_master/scaler/system_function_pod_manager/system_function_pod_manager.h b/functionsystem/src/function_master/scaler/system_function_pod_manager/system_function_pod_manager.h index 986d612..ccc37d0 100644 --- a/functionsystem/src/function_master/scaler/system_function_pod_manager/system_function_pod_manager.h +++ b/functionsystem/src/function_master/scaler/system_function_pod_manager/system_function_pod_manager.h @@ -20,6 +20,8 @@ #include #include +#include "utils/string_utils.hpp" + #include "common/kube_client/model/pod/v1_pod.h" #include "common/status/status.h" diff --git a/functionsystem/src/function_master/system_function_loader/CMakeLists.txt b/functionsystem/src/function_master/system_function_loader/CMakeLists.txt index 769fbc2..7b19e43 100644 --- a/functionsystem/src/function_master/system_function_loader/CMakeLists.txt +++ b/functionsystem/src/function_master/system_function_loader/CMakeLists.txt @@ -11,7 +11,6 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(${SYSTEM_FUNCTION_LOADER_LIB} STATIC ${SYSTEM_FUNCTION_LOADER_SRCS}) -set_target_properties(${SYSTEM_FUNCTION_LOADER_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${SYSTEM_FUNCTION_LOADER_LIB} PRIVATE -fPIE) target_include_directories(${SYSTEM_FUNCTION_LOADER_LIB} PRIVATE diff --git a/functionsystem/src/function_master/system_function_loader/bootstrap_actor.cpp b/functionsystem/src/function_master/system_function_loader/bootstrap_actor.cpp index c20abd7..9bd4c1e 100644 --- a/functionsystem/src/function_master/system_function_loader/bootstrap_actor.cpp +++ b/functionsystem/src/function_master/system_function_loader/bootstrap_actor.cpp @@ -35,16 +35,6 @@ #include "utils/string_utils.hpp" namespace functionsystem::system_function_loader { -const std::string BOOTSTRAP_CONFIG_PATH = "/home/sn/function/config/system-function-config.json"; // NOLINT -const std::string METAFILE_WATCH_PATH = "/home/sn/function/system-function-meta"; -const std::string SYSFUNCTION_CONFIG_KEY = "/faas/system-function/config"; -const uint64_t SENDARGS_TIMEOUT_MS = 120000; -const uint64_t UPGRADE_TIMEOUT_MS = 60000; -const uint32_t WAIT_INSTANCE_MAX_TIMES = 60; -const int MAX_RETRY_TIMES = 5; -const int RETRY_TIMEOUT_MS = 10000; -const int MAX_SYSFUNCTION_META_INFO = 6; - using namespace functionsystem::explorer; BootstrapActor::BootstrapActor(const std::shared_ptr &metaClient, diff --git a/functionsystem/src/function_master/system_function_loader/bootstrap_actor.h b/functionsystem/src/function_master/system_function_loader/bootstrap_actor.h index c69f664..1a228c5 100644 --- a/functionsystem/src/function_master/system_function_loader/bootstrap_actor.h +++ b/functionsystem/src/function_master/system_function_loader/bootstrap_actor.h @@ -34,6 +34,15 @@ namespace functionsystem::system_function_loader { const uint64_t WAIT_KILLINSTANCE_MS = 3000; const uint64_t WAIT_STARTINSTANCE_MS = 10000; const uint64_t WAIT_UPDATE_CONFIGMAP_MS = 1000; +const std::string BOOTSTRAP_CONFIG_PATH = "/home/sn/function/config/system-function-config.json"; // NOLINT +const std::string METAFILE_WATCH_PATH = "/home/sn/function/system-function-meta"; +const std::string SYSFUNCTION_CONFIG_KEY = "/faas/system-function/config"; +const uint64_t SENDARGS_TIMEOUT_MS = 120000; +const uint64_t UPGRADE_TIMEOUT_MS = 60000; +const uint32_t WAIT_INSTANCE_MAX_TIMES = 60; +const int MAX_RETRY_TIMES = 5; +const int RETRY_TIMEOUT_MS = 10000; +const int MAX_SYSFUNCTION_META_INFO = 6; struct FunctionConfig { std::string tenantID; diff --git a/functionsystem/src/function_proxy/busproxy/CMakeLists.txt b/functionsystem/src/function_proxy/busproxy/CMakeLists.txt index 75862b8..0841255 100644 --- a/functionsystem/src/function_proxy/busproxy/CMakeLists.txt +++ b/functionsystem/src/function_proxy/busproxy/CMakeLists.txt @@ -26,8 +26,6 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/startup STARTUP_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/memory_monitor MONITOR_SRCS) add_library(busproxy_lib STATIC "") -set_target_properties(busproxy_lib PROPERTIES UNITY_BUILD ON) -set_target_properties(busproxy_lib PROPERTIES UNITY_BUILD_BATCH_SIZE 9) # only 9 files, set in one batch target_compile_options(busproxy_lib PRIVATE -fPIE) target_sources(busproxy_lib PRIVATE ${INSTANCE_PROXY_SRCS} ${INSTANCE_VIEW_SRCS} ${INVOCATION_SRCS} ${REGISTRY_SRCS} ${STARTUP_SRCS} ${MONITOR_SRCS}) target_link_libraries(busproxy_lib ${LITEBUS_LIB} meta_store_client meta_store_adapter utils) diff --git a/functionsystem/src/function_proxy/common/CMakeLists.txt b/functionsystem/src/function_proxy/common/CMakeLists.txt index 7ef9999..6d1ba69 100644 --- a/functionsystem/src/function_proxy/common/CMakeLists.txt +++ b/functionsystem/src/function_proxy/common/CMakeLists.txt @@ -14,7 +14,6 @@ project(function_proxy_common) add_library(function_proxy_common_lib STATIC "") -set_target_properties(function_proxy_common_lib PROPERTIES UNITY_BUILD ON) target_compile_options(function_proxy_common_lib PRIVATE -fPIE) target_include_directories(function_proxy_common_lib PUBLIC ${CMAKE_CURRENT_LIST_DIR}) diff --git a/functionsystem/src/function_proxy/local_scheduler/CMakeLists.txt b/functionsystem/src/function_proxy/local_scheduler/CMakeLists.txt index c95e5b7..07efdcf 100644 --- a/functionsystem/src/function_proxy/local_scheduler/CMakeLists.txt +++ b/functionsystem/src/function_proxy/local_scheduler/CMakeLists.txt @@ -17,8 +17,6 @@ project(local_scheduler) set_property(GLOBAL PROPERTY SRC_LOCAL_SCHEDULER_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}") add_library(local_scheduler_lib STATIC "") -set_target_properties(local_scheduler_lib PROPERTIES UNITY_BUILD ON) -set_target_properties(local_scheduler_lib PROPERTIES UNITY_BUILD_BATCH_SIZE 4) target_compile_options(local_scheduler_lib PRIVATE -fPIE) target_include_directories(local_scheduler_lib PUBLIC ${CMAKE_CURRENT_LIST_DIR}) diff --git a/functionsystem/src/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker.cpp b/functionsystem/src/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker.cpp index 5d3b8e2..0c1f975 100644 --- a/functionsystem/src/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker.cpp +++ b/functionsystem/src/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker.cpp @@ -17,6 +17,8 @@ #include "async/asyncafter.hpp" #include "common/logs/logging.h" +#include "common/constants/constants.h" +#include "utils/os_utils.hpp" namespace functionsystem::local_scheduler { diff --git a/functionsystem/src/iam_server/CMakeLists.txt b/functionsystem/src/iam_server/CMakeLists.txt index 04310fc..ef28be7 100644 --- a/functionsystem/src/iam_server/CMakeLists.txt +++ b/functionsystem/src/iam_server/CMakeLists.txt @@ -24,7 +24,6 @@ list(REMOVE_ITEM IAM_SERVER_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp") # exclu include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(iam_server_lib STATIC ${IAM_SERVER_SRCS}) -set_target_properties(iam_server_lib PROPERTIES UNITY_BUILD ON) target_compile_options(iam_server_lib PRIVATE -fPIE) target_link_libraries(iam_server_lib ${etcdapi_LIB_A} kube_client diff --git a/functionsystem/src/meta_store/client/cpp/CMakeLists.txt b/functionsystem/src/meta_store/client/cpp/CMakeLists.txt index a3f238f..ba33a3b 100644 --- a/functionsystem/src/meta_store/client/cpp/CMakeLists.txt +++ b/functionsystem/src/meta_store/client/cpp/CMakeLists.txt @@ -31,7 +31,6 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}/include/meta_store_client/maintena include_directories(${CMAKE_CURRENT_LIST_DIR}/include/meta_store_client/utils) add_library(meta_store_client STATIC "") -set_target_properties(meta_store_client PROPERTIES UNITY_BUILD ON) file(GLOB_RECURSE META_STORE_CLIENT_SRCS src/*.cpp) diff --git a/functionsystem/src/meta_store/client/cpp/src/meta_store_client/meta_store_client_mgr.cpp b/functionsystem/src/meta_store/client/cpp/src/meta_store_client/meta_store_client_mgr.cpp index 011a21a..0b23378 100644 --- a/functionsystem/src/meta_store/client/cpp/src/meta_store_client/meta_store_client_mgr.cpp +++ b/functionsystem/src/meta_store/client/cpp/src/meta_store_client/meta_store_client_mgr.cpp @@ -28,6 +28,7 @@ #include "meta_store_client/lease/meta_store_lease_client_strategy.h" #include "meta_store_client/maintenance/etcd_maintenance_client_strategy.h" #include "meta_store_client/maintenance/meta_store_maintenance_client_strategy.h" +#include "common/utils/random_number.h" namespace functionsystem { diff --git a/functionsystem/src/meta_store/server/src/CMakeLists.txt b/functionsystem/src/meta_store/server/src/CMakeLists.txt index 0ab6a86..f9dd0cb 100644 --- a/functionsystem/src/meta_store/server/src/CMakeLists.txt +++ b/functionsystem/src/meta_store/server/src/CMakeLists.txt @@ -28,7 +28,6 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(${META_STORE_LIB} STATIC ${META_STORE_SRCS}) -set_target_properties(${META_STORE_LIB} PROPERTIES UNITY_BUILD ON) target_compile_options(${META_STORE_LIB} PRIVATE -fPIE) add_dependencies(${META_STORE_LIB} meta_store_client http explorer) diff --git a/functionsystem/src/runtime_manager/CMakeLists.txt b/functionsystem/src/runtime_manager/CMakeLists.txt index 175d2e3..798e16c 100644 --- a/functionsystem/src/runtime_manager/CMakeLists.txt +++ b/functionsystem/src/runtime_manager/CMakeLists.txt @@ -23,7 +23,6 @@ list(REMOVE_ITEM RUNTIME_MANAGER_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp") # include_directories(${CMAKE_CURRENT_LIST_DIR}) add_library(runtime_manager_lib STATIC ${RUNTIME_MANAGER_SRCS}) -set_target_properties(runtime_manager_lib PROPERTIES UNITY_BUILD ON) target_compile_options(runtime_manager_lib PRIVATE -fPIE) add_subdirectory(executor) diff --git a/functionsystem/src/runtime_manager/config/flags.cpp b/functionsystem/src/runtime_manager/config/flags.cpp index ca66bcc..4d8e310 100644 --- a/functionsystem/src/runtime_manager/config/flags.cpp +++ b/functionsystem/src/runtime_manager/config/flags.cpp @@ -19,6 +19,7 @@ #include #include "common/utils/param_check.h" +#include "common/utils/exec_utils.h" #include "utils/os_utils.hpp" namespace functionsystem::runtime_manager { -- Gitee